Upgrade versions of vendored libs

vendor/github.com/cockroachdb/pebble/sstable/block.go

@@ -8,10 +8,12 @@ import (
 	"encoding/binary"
 	"unsafe"
 
+	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
+	"github.com/cockroachdb/pebble/internal/manual"
 	"github.com/cockroachdb/pebble/internal/rangedel"
 	"github.com/cockroachdb/pebble/internal/rangekey"
 )
@@ -30,11 +32,33 @@ type blockWriter struct {
 	nEntries        int
 	nextRestart     int
 	buf             []byte
-	restarts        []uint32
-	curKey          []byte
-	curValue        []byte
-	prevKey         []byte
-	tmp             [4]byte
+	// For datablocks in TableFormatPebblev3, we steal the most significant bit
+	// in restarts for encoding setHasSameKeyPrefixSinceLastRestart. This leaves
+	// us with 31 bits, which is more than enough (no one needs > 2GB blocks).
+	// Typically, restarts occur every 16 keys, and by storing this bit with the
+	// restart, we can optimize for the case where a user wants to skip to the
+	// next prefix which happens to be in the same data block, but is > 16 keys
+	// away. We have seen production situations with 100+ versions per MVCC key
+	// (which share the same prefix). Additionally, for such writers, the prefix
+	// compression of the key, that shares the key with the preceding key, is
+	// limited to the prefix part of the preceding key -- this ensures that when
+	// doing NPrefix (see blockIter) we don't need to assemble the full key
+	// for each step since by limiting the length of the shared key we are
+	// ensuring that any of the keys with the same prefix can be used to
+	// assemble the full key when the prefix does change.
+	restarts []uint32
+	curKey   []byte
+	// curValue excludes the optional prefix provided to
+	// storeWithOptionalValuePrefix.
+	curValue []byte
+	prevKey  []byte
+	tmp      [4]byte
+	// We don't know the state of the sets that were at the end of the previous
+	// block, so this is initially 0. It may be true for the second and later
+	// restarts in a block. Not having inter-block information is fine since we
+	// will optimize by stepping through restarts only within the same block.
+	// Note that the first restart is the first key in the block.
+	setHasSameKeyPrefixSinceLastRestart bool
 }
 
 func (w *blockWriter) clear() {
@@ -47,17 +71,41 @@ func (w *blockWriter) clear() {
 	}
 }
 
-func (w *blockWriter) store(keySize int, value []byte) {
+// MaximumBlockSize is an extremely generous maximum block size of 256MiB. We
+// explicitly place this limit to reserve a few bits in the restart for
+// internal use.
+const MaximumBlockSize = 1 << 28
+const setHasSameKeyPrefixRestartMask uint32 = 1 << 31
+const restartMaskLittleEndianHighByteWithoutSetHasSamePrefix byte = 0b0111_1111
+const restartMaskLittleEndianHighByteOnlySetHasSamePrefix byte = 0b1000_0000
+
+// If !addValuePrefix, the valuePrefix is ignored.
+func (w *blockWriter) storeWithOptionalValuePrefix(
+	keySize int,
+	value []byte,
+	maxSharedKeyLen int,
+	addValuePrefix bool,
+	valuePrefix valuePrefix,
+	setHasSameKeyPrefix bool,
+) {
 	shared := 0
+	if !setHasSameKeyPrefix {
+		w.setHasSameKeyPrefixSinceLastRestart = false
+	}
 	if w.nEntries == w.nextRestart {
 		w.nextRestart = w.nEntries + w.restartInterval
-		w.restarts = append(w.restarts, uint32(len(w.buf)))
+		restart := uint32(len(w.buf))
+		if w.setHasSameKeyPrefixSinceLastRestart {
+			restart = restart | setHasSameKeyPrefixRestartMask
+		}
+		w.setHasSameKeyPrefixSinceLastRestart = true
+		w.restarts = append(w.restarts, restart)
 	} else {
 		// TODO(peter): Manually inlined version of base.SharedPrefixLen(). This
 		// is 3% faster on BenchmarkWriter on go1.16. Remove if future versions
 		// show this to not be a performance win. For now, functions that use of
 		// unsafe cannot be inlined.
-		n := len(w.curKey)
+		n := maxSharedKeyLen
 		if n > len(w.prevKey) {
 			n = len(w.prevKey)
 		}
@@ -72,7 +120,11 @@ func (w *blockWriter) store(keySize int, value []byte) {
 		}
 	}
 
-	needed := 3*binary.MaxVarintLen32 + len(w.curKey[shared:]) + len(value)
+	lenValuePlusOptionalPrefix := len(value)
+	if addValuePrefix {
+		lenValuePlusOptionalPrefix++
+	}
+	needed := 3*binary.MaxVarintLen32 + len(w.curKey[shared:]) + lenValuePlusOptionalPrefix
 	n := len(w.buf)
 	if cap(w.buf) < n+needed {
 		newCap := 2 * cap(w.buf)
@@ -114,7 +166,7 @@ func (w *blockWriter) store(keySize int, value []byte) {
 	}
 
 	{
-		x := uint32(len(value))
+		x := uint32(lenValuePlusOptionalPrefix)
 		for x >= 0x80 {
 			w.buf[n] = byte(x) | 0x80
 			x >>= 7
@@ -125,6 +177,10 @@ func (w *blockWriter) store(keySize int, value []byte) {
 	}
 
 	n += copy(w.buf[n:], w.curKey[shared:])
+	if addValuePrefix {
+		w.buf[n : n+1][0] = byte(valuePrefix)
+		n++
+	}
 	n += copy(w.buf[n:], value)
 	w.buf = w.buf[:n]
 
@@ -134,6 +190,19 @@ func (w *blockWriter) store(keySize int, value []byte) {
 }
 
 func (w *blockWriter) add(key InternalKey, value []byte) {
+	w.addWithOptionalValuePrefix(
+		key, value, len(key.UserKey), false, 0, false)
+}
+
+// Callers that always set addValuePrefix to false should use add() instead.
+func (w *blockWriter) addWithOptionalValuePrefix(
+	key InternalKey,
+	value []byte,
+	maxSharedKeyLen int,
+	addValuePrefix bool,
+	valuePrefix valuePrefix,
+	setHasSameKeyPrefix bool,
+) {
 	w.curKey, w.prevKey = w.prevKey, w.curKey
 
 	size := key.Size()
@@ -143,7 +212,8 @@ func (w *blockWriter) add(key InternalKey, value []byte) {
 	w.curKey = w.curKey[:size]
 	key.Encode(w.curKey)
 
-	w.store(size, value)
+	w.storeWithOptionalValuePrefix(
+		size, value, maxSharedKeyLen, addValuePrefix, valuePrefix, setHasSameKeyPrefix)
 }
 
 func (w *blockWriter) finish() []byte {
@@ -211,6 +281,36 @@ type blockEntry struct {
 // range keys since there is only a single range deletion and range key block
 // per sstable and the blockIter will not release the bytes for the block until
 // it is closed.
+//
+// Note on why blockIter knows about lazyValueHandling:
+//
+// blockIter's positioning functions (that return a LazyValue), are too
+// complex to inline even prior to lazyValueHandling. blockIter.Next and
+// blockIter.First were by far the cheapest and had costs 195 and 180
+// respectively, which exceeds the budget of 80. We initially tried to keep
+// the lazyValueHandling logic out of blockIter by wrapping it with a
+// lazyValueDataBlockIter. singleLevelIter and twoLevelIter would use this
+// wrapped iter. The functions in lazyValueDataBlockIter were simple, in that
+// they called the corresponding blockIter func and then decided whether the
+// value was in fact in-place (so return immediately) or needed further
+// handling. But these also turned out too costly for mid-stack inlining since
+// simple calls like the following have a high cost that is barely under the
+// budget of 80
+//
+//	k, v := i.data.SeekGE(key, flags)  // cost 74
+//	k, v := i.data.Next()              // cost 72
+//
+// We have 2 options for minimizing performance regressions:
+//   - Include the lazyValueHandling logic in the already non-inlineable
+//     blockIter functions: Since most of the time is spent in data block iters,
+//     it is acceptable to take the small hit of unnecessary branching (which
+//     hopefully branch prediction will predict correctly) for other kinds of
+//     blocks.
+//   - Duplicate the logic of singleLevelIterator and twoLevelIterator for the
+//     v3 sstable and only use the aforementioned lazyValueDataBlockIter for a
+//     v3 sstable. We would want to manage these copies via code generation.
+//
+// We have picked the first option here.
 type blockIter struct {
 	cmp Compare
 	// offset is the byte index that marks where the current key/value is
@@ -248,6 +348,9 @@ type blockIter struct {
 	// val contains the value the iterator is currently pointed at. If non-nil,
 	// this points to a slice of the block data.
 	val []byte
+	// lazyValue is val turned into a LazyValue, whenever a positioning method
+	// returns a non-nil key-value pair.
+	lazyValue base.LazyValue
 	// ikey contains the decoded InternalKey the iterator is currently pointed
 	// at. Note that the memory backing ikey.UserKey is either data stored
 	// directly in the block, fullKey, or cachedBuf. The key stability guarantee
@@ -273,7 +376,11 @@ type blockIter struct {
 	cacheHandle cache.Handle
 	// The first key in the block. This is used by the caller to set bounds
 	// for block iteration for already loaded blocks.
-	firstKey InternalKey
+	firstKey          InternalKey
+	lazyValueHandling struct {
+		vbr            *valueBlockReader
+		hasValuePrefix bool
+	}
 }
 
 // blockIter implements the base.InternalIterator interface.
@@ -413,6 +520,7 @@ func (i *blockIter) readEntry() {
 	}
 
 	unsharedKey := getBytes(ptr, int(unshared))
+	// TODO(sumeer): move this into the else block below.
 	i.fullKey = append(i.fullKey[:shared], unsharedKey...)
 	if shared == 0 {
 		// Provide stability for the key across positioning calls if the key
@@ -553,7 +661,7 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba
 		for index < upper {
 			h := int32(uint(index+upper) >> 1) // avoid overflow when computing h
 			// index ≤ h < upper
-			offset := int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*h:]))
+			offset := decodeRestart(i.data[i.restarts+4*h:])
 			// For a restart point, there are 0 bytes shared with the previous key.
 			// The varint encoding of 0 occupies 1 byte.
 			ptr := unsafe.Pointer(uintptr(i.ptr) + uintptr(offset+1))
@@ -621,18 +729,33 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba
 	// 0, then all keys in this block are larger than the key sought, and offset
 	// remains at zero.
 	if index > 0 {
-		i.offset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(index-1):]))
+		i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
 	}
 	i.readEntry()
 	i.decodeInternalKey(i.key)
 
 	// Iterate from that restart point to somewhere >= the key sought.
-	for ; i.valid(); i.Next() {
+	if !i.valid() {
+		return nil, base.LazyValue{}
+	}
+	if base.InternalCompare(i.cmp, i.ikey, ikey) >= 0 {
+		// Initialize i.lazyValue
+		if !i.lazyValueHandling.hasValuePrefix ||
+			base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+			i.lazyValue = base.MakeInPlaceValue(i.val)
+		} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+			i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+		} else {
+			i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+		}
+		return &i.ikey, i.lazyValue
+	}
+	for i.Next(); i.valid(); i.Next() {
 		if base.InternalCompare(i.cmp, i.ikey, ikey) >= 0 {
-			return &i.ikey, base.MakeInPlaceValue(i.val)
+			// i.Next() has already initialized i.lazyValue.
+			return &i.ikey, i.lazyValue
 		}
 	}
-
 	return nil, base.LazyValue{}
 }
 
@@ -666,7 +789,7 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
 		for index < upper {
 			h := int32(uint(index+upper) >> 1) // avoid overflow when computing h
 			// index ≤ h < upper
-			offset := int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*h:]))
+			offset := decodeRestart(i.data[i.restarts+4*h:])
 			// For a restart point, there are 0 bytes shared with the previous key.
 			// The varint encoding of 0 occupies 1 byte.
 			ptr := unsafe.Pointer(uintptr(i.ptr) + uintptr(offset+1))
@@ -733,9 +856,9 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
 	// index-1 will be the largest whose key is < the key sought.
 	targetOffset := i.restarts
 	if index > 0 {
-		i.offset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(index-1):]))
+		i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
 		if index < i.numRestarts {
-			targetOffset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(index):]))
+			targetOffset = decodeRestart(i.data[i.restarts+4*(index):])
 		}
 	} else if index == 0 {
 		// If index == 0 then all keys in this block are larger than the key
@@ -757,11 +880,12 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
 
 		if i.cmp(i.ikey.UserKey, ikey.UserKey) >= 0 {
 			// The current key is greater than or equal to our search key. Back up to
-			// the previous key which was less than our search key. Note that his for
+			// the previous key which was less than our search key. Note that this for
 			// loop will execute at least once with this if-block not being true, so
 			// the key we are backing up to is the last one this loop cached.
 			i.Prev()
-			return &i.ikey, base.MakeInPlaceValue(i.val)
+			// i.Prev() has already initialized i.lazyValue.
+			return &i.ikey, i.lazyValue
 		}
 
 		if i.nextOffset >= targetOffset {
@@ -779,7 +903,15 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
 	if !i.valid() {
 		return nil, base.LazyValue{}
 	}
-	return &i.ikey, base.MakeInPlaceValue(i.val)
+	if !i.lazyValueHandling.hasValuePrefix ||
+		base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+		i.lazyValue = base.MakeInPlaceValue(i.val)
+	} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+		i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+	} else {
+		i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+	}
+	return &i.ikey, i.lazyValue
 }
 
 // First implements internalIterator.First, as documented in the pebble
@@ -792,13 +924,27 @@ func (i *blockIter) First() (*InternalKey, base.LazyValue) {
 	i.clearCache()
 	i.readEntry()
 	i.decodeInternalKey(i.key)
-	return &i.ikey, base.MakeInPlaceValue(i.val)
+	if !i.lazyValueHandling.hasValuePrefix ||
+		base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+		i.lazyValue = base.MakeInPlaceValue(i.val)
+	} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+		i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+	} else {
+		i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+	}
+	return &i.ikey, i.lazyValue
+}
+
+func decodeRestart(b []byte) int32 {
+	_ = b[3] // bounds check hint to compiler; see golang.org/issue/14808
+	return int32(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 |
+		uint32(b[3]&restartMaskLittleEndianHighByteWithoutSetHasSamePrefix)<<24)
 }
 
 // Last implements internalIterator.Last, as documented in the pebble package.
 func (i *blockIter) Last() (*InternalKey, base.LazyValue) {
 	// Seek forward from the last restart point.
-	i.offset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(i.numRestarts-1):]))
+	i.offset = decodeRestart(i.data[i.restarts+4*(i.numRestarts-1):])
 	if !i.valid() {
 		return nil, base.LazyValue{}
 	}
@@ -813,7 +959,15 @@ func (i *blockIter) Last() (*InternalKey, base.LazyValue) {
 	}
 
 	i.decodeInternalKey(i.key)
-	return &i.ikey, base.MakeInPlaceValue(i.val)
+	if !i.lazyValueHandling.hasValuePrefix ||
+		base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+		i.lazyValue = base.MakeInPlaceValue(i.val)
+	} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+		i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+	} else {
+		i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+	}
+	return &i.ikey, i.lazyValue
 }
 
 // Next implements internalIterator.Next, as documented in the pebble
@@ -850,7 +1004,307 @@ func (i *blockIter) Next() (*InternalKey, base.LazyValue) {
 		i.ikey.Trailer = uint64(InternalKeyKindInvalid)
 		i.ikey.UserKey = nil
 	}
-	return &i.ikey, base.MakeInPlaceValue(i.val)
+	if !i.lazyValueHandling.hasValuePrefix ||
+		base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+		i.lazyValue = base.MakeInPlaceValue(i.val)
+	} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+		i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+	} else {
+		i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+	}
+	return &i.ikey, i.lazyValue
+}
+
+// nextPrefix is used for implementing NPrefix.
+func (i *blockIter) nextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
+	if i.lazyValueHandling.hasValuePrefix {
+		return i.nextPrefixV3(succKey)
+	}
+	const nextsBeforeSeek = 3
+	k, v := i.Next()
+	for j := 1; k != nil && i.cmp(k.UserKey, succKey) < 0; j++ {
+		if j >= nextsBeforeSeek {
+			return i.SeekGE(succKey, base.SeekGEFlagsNone)
+		}
+		k, v = i.Next()
+	}
+	return k, v
+}
+
+func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue) {
+	// Doing nexts that involve a key comparison can be expensive (and the cost
+	// depends on the key length), so we use the same threshold of 3 that we use
+	// for TableFormatPebblev2 in blockIter.nextPrefix above. The next fast path
+	// that looks at setHasSamePrefix takes ~5ns per key, which is ~150x faster
+	// than doing a SeekGE within the block, so we do this 16 times
+	// (~5ns*16=80ns), and then switch to looking at restarts. Doing the binary
+	// search for the restart consumes > 100ns. If the number of versions is >
+	// 17, we will increment nextFastCount to 17, then do a binary search, and
+	// on average need to find a key between two restarts, so another 8 steps
+	// corresponding to nextFastCount, for a mean total of 17 + 8 = 25 such
+	// steps.
+	//
+	// TODO(sumeer): use the configured restartInterval for the sstable when it
+	// was written (which we don't currently store) instead of the default value
+	// of 16.
+	const nextCmpThresholdBeforeSeek = 3
+	const nextFastThresholdBeforeRestarts = 16
+	nextCmpCount := 0
+	nextFastCount := 0
+	usedRestarts := false
+	// INVARIANT: blockIter is valid.
+	if invariants.Enabled && !i.valid() {
+		panic(errors.AssertionFailedf("nextPrefixV3 called on invalid blockIter"))
+	}
+	prevKeyIsSet := i.ikey.Kind() == InternalKeyKindSet
+	for {
+		i.offset = i.nextOffset
+		if !i.valid() {
+			return nil, base.LazyValue{}
+		}
+		// Need to decode the length integers, so we can compute nextOffset.
+		ptr := unsafe.Pointer(uintptr(i.ptr) + uintptr(i.offset))
+		// This is an ugly performance hack. Reading entries from blocks is one of
+		// the inner-most routines and decoding the 3 varints per-entry takes
+		// significant time. Neither go1.11 or go1.12 will inline decodeVarint for
+		// us, so we do it manually. This provides a 10-15% performance improvement
+		// on blockIter benchmarks on both go1.11 and go1.12.
+		//
+		// TODO(peter): remove this hack if go:inline is ever supported.
+
+		// Decode the shared key length integer.
+		var shared uint32
+		if a := *((*uint8)(ptr)); a < 128 {
+			shared = uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 1)
+		} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
+			shared = uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 2)
+		} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
+			shared = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 3)
+		} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
+			shared = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 4)
+		} else {
+			d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
+			shared = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 5)
+		}
+		// Decode the unshared key length integer.
+		var unshared uint32
+		if a := *((*uint8)(ptr)); a < 128 {
+			unshared = uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 1)
+		} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
+			unshared = uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 2)
+		} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
+			unshared = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 3)
+		} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
+			unshared = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 4)
+		} else {
+			d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
+			unshared = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 5)
+		}
+		// Decode the value length integer.
+		var value uint32
+		if a := *((*uint8)(ptr)); a < 128 {
+			value = uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 1)
+		} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
+			value = uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 2)
+		} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
+			value = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 3)
+		} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
+			value = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 4)
+		} else {
+			d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
+			value = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 5)
+		}
+		// The starting position of the value.
+		valuePtr := unsafe.Pointer(uintptr(ptr) + uintptr(unshared))
+		i.nextOffset = int32(uintptr(valuePtr)-uintptr(i.ptr)) + int32(value)
+		if invariants.Enabled && unshared < 8 {
+			// This should not happen since only the key prefix is shared, so even
+			// if the prefix length is the same as the user key length, the unshared
+			// will include the trailer.
+			panic(errors.AssertionFailedf("unshared %d is too small", unshared))
+		}
+		// The trailer is written in little endian, so the key kind is the first
+		// byte in the trailer that is encoded in the slice [unshared-8:unshared].
+		keyKind := InternalKeyKind((*[manual.MaxArrayLen]byte)(ptr)[unshared-8])
+		prefixChanged := false
+		if keyKind == InternalKeyKindSet {
+			if invariants.Enabled && value == 0 {
+				panic(errors.AssertionFailedf("value is of length 0, but we expect a valuePrefix"))
+			}
+			valPrefix := *((*valuePrefix)(valuePtr))
+			if setHasSamePrefix(valPrefix) {
+				// Fast-path. No need to assemble i.fullKey, or update i.key. We know
+				// that subsequent keys will not have a shared length that is greater
+				// than the prefix of the current key, which is also the prefix of
+				// i.key. Since we are continuing to iterate, we don't need to
+				// initialize i.ikey and i.lazyValue (these are initialized before
+				// returning).
+				nextFastCount++
+				if nextFastCount > nextFastThresholdBeforeRestarts {
+					if usedRestarts {
+						// Exhausted iteration budget. This will never happen unless
+						// someone is using a restart interval > 16. It is just to guard
+						// against long restart intervals causing too much iteration.
+						break
+					}
+					// Haven't used restarts yet, so find the first restart at or beyond
+					// the current offset.
+					targetOffset := i.offset
+					var index int32
+					{
+						// NB: manually inlined sort.Sort is ~5% faster.
+						//
+						// f defined for a restart point is true iff the offset >=
+						// targetOffset.
+						// Define f(-1) == false and f(i.numRestarts) == true.
+						// Invariant: f(index-1) == false, f(upper) == true.
+						upper := i.numRestarts
+						for index < upper {
+							h := int32(uint(index+upper) >> 1) // avoid overflow when computing h
+							// index ≤ h < upper
+							offset := decodeRestart(i.data[i.restarts+4*h:])
+							if offset < targetOffset {
+								index = h + 1 // preserves f(index-1) == false
+							} else {
+								upper = h // preserves f(upper) == true
+							}
+						}
+						// index == upper, f(index-1) == false, and f(upper) (= f(index)) == true
+						// => answer is index.
+					}
+					usedRestarts = true
+					nextFastCount = 0
+					if index == i.numRestarts {
+						// Already past the last real restart, so iterate a bit more until
+						// we are done with the block.
+						continue
+					}
+					// Have some real restarts after index. NB: index is the first
+					// restart at or beyond the current offset.
+					startingIndex := index
+					for index != i.numRestarts &&
+						// The restart at index is 4 bytes written in little endian format
+						// starting at i.restart+4*index. The 0th byte is the least
+						// significant and the 3rd byte is the most significant. Since the
+						// most significant bit of the 3rd byte is what we use for
+						// encoding the set-has-same-prefix information, the indexing
+						// below has +3.
+						i.data[i.restarts+4*index+3]&restartMaskLittleEndianHighByteOnlySetHasSamePrefix != 0 {
+						// We still have the same prefix, so move to the next restart.
+						index++
+					}
+					// index is the first restart that did not have the same prefix.
+					if index != startingIndex {
+						// Managed to skip past at least one restart. Resume iteration
+						// from index-1. Since nextFastCount has been reset to 0, we
+						// should be able to iterate to the next prefix.
+						i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
+						i.readEntry()
+					}
+					// Else, unable to skip past any restart. Resume iteration. Since
+					// nextFastCount has been reset to 0, we should be able to iterate
+					// to the next prefix.
+					continue
+				}
+				continue
+			} else if prevKeyIsSet {
+				prefixChanged = true
+			}
+		} else {
+			prevKeyIsSet = false
+		}
+		// Slow-path cases:
+		// - (Likely) The prefix has changed.
+		// - (Unlikely) The prefix has not changed.
+		// We assemble the key etc. under the assumption that it is the likely
+		// case.
+		unsharedKey := getBytes(ptr, int(unshared))
+		// TODO(sumeer): move this into the else block below. This is a bit tricky
+		// since the current logic assumes we have always copied the latest key
+		// into fullKey, which is why when we get to the next key we can (a)
+		// access i.fullKey[:shared], (b) append only the unsharedKey to
+		// i.fullKey. For (a), we can access i.key[:shared] since that memory is
+		// valid (even if unshared). For (b), we will need to remember whether
+		// i.key refers to i.fullKey or not, and can append the unsharedKey only
+		// in the former case and for the latter case need to copy the shared part
+		// too. This same comment applies to the other place where we can do this
+		// optimization, in readEntry().
+		i.fullKey = append(i.fullKey[:shared], unsharedKey...)
+		i.val = getBytes(valuePtr, int(value))
+		if shared == 0 {
+			// Provide stability for the key across positioning calls if the key
+			// doesn't share a prefix with the previous key. This removes requiring the
+			// key to be copied if the caller knows the block has a restart interval of
+			// 1. An important example of this is range-del blocks.
+			i.key = unsharedKey
+		} else {
+			i.key = i.fullKey
+		}
+		// Manually inlined version of i.decodeInternalKey(i.key).
+		if n := len(i.key) - 8; n >= 0 {
+			i.ikey.Trailer = binary.LittleEndian.Uint64(i.key[n:])
+			i.ikey.UserKey = i.key[:n:n]
+			if i.globalSeqNum != 0 {
+				i.ikey.SetSeqNum(i.globalSeqNum)
+			}
+		} else {
+			i.ikey.Trailer = uint64(InternalKeyKindInvalid)
+			i.ikey.UserKey = nil
+		}
+		nextCmpCount++
+		if invariants.Enabled && prefixChanged && i.cmp(i.ikey.UserKey, succKey) < 0 {
+			panic(errors.AssertionFailedf("prefix should have changed but %x < %x",
+				i.ikey.UserKey, succKey))
+		}
+		if prefixChanged || i.cmp(i.ikey.UserKey, succKey) >= 0 {
+			// Prefix has changed.
+			if invariants.Enabled && !i.lazyValueHandling.hasValuePrefix {
+				panic(errors.AssertionFailedf("nextPrefixV3 being run for non-v3 sstable"))
+			}
+			if base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+				i.lazyValue = base.MakeInPlaceValue(i.val)
+			} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+				i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+			} else {
+				i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+			}
+			return &i.ikey, i.lazyValue
+		}
+		// Else prefix has not changed.
+
+		if nextCmpCount >= nextCmpThresholdBeforeSeek {
+			break
+		}
+	}
+	return i.SeekGE(succKey, base.SeekGEFlagsNone)
+}
+
+// NextPrefix implements (base.InternalIterator).NextPrefix
+func (i *blockIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
+	const nextsBeforeSeek = 3
+	k, v := i.Next()
+	for j := 1; k != nil && i.cmp(k.UserKey, succKey) < 0; j++ {
+		if j >= nextsBeforeSeek {
+			return i.SeekGE(succKey, base.SeekGEFlagsNone)
+		}
+		k, v = i.Next()
+	}
+	return k, v
 }
 
 // Prev implements internalIterator.Prev, as documented in the pebble
@@ -874,7 +1328,15 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
 			i.ikey.UserKey = nil
 		}
 		i.cached = i.cached[:n]
-		return &i.ikey, base.MakeInPlaceValue(i.val)
+		if !i.lazyValueHandling.hasValuePrefix ||
+			base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+			i.lazyValue = base.MakeInPlaceValue(i.val)
+		} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+			i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+		} else {
+			i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+		}
+		return &i.ikey, i.lazyValue
 	}
 
 	i.clearCache()
@@ -896,7 +1358,7 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
 		for index < upper {
 			h := int32(uint(index+upper) >> 1) // avoid overflow when computing h
 			// index ≤ h < upper
-			offset := int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*h:]))
+			offset := decodeRestart(i.data[i.restarts+4*h:])
 			if offset < targetOffset {
 				index = h + 1 // preserves f(i-1) == false
 			} else {
@@ -909,7 +1371,7 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
 
 	i.offset = 0
 	if index > 0 {
-		i.offset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(index-1):]))
+		i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
 	}
 
 	i.readEntry()
@@ -921,7 +1383,15 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
 	}
 
 	i.decodeInternalKey(i.key)
-	return &i.ikey, base.MakeInPlaceValue(i.val)
+	if !i.lazyValueHandling.hasValuePrefix ||
+		base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+		i.lazyValue = base.MakeInPlaceValue(i.val)
+	} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+		i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+	} else {
+		i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+	}
+	return &i.ikey, i.lazyValue
 }
 
 // Key implements internalIterator.Key, as documented in the pebble package.
@@ -930,7 +1400,7 @@ func (i *blockIter) Key() *InternalKey {
 }
 
 func (i *blockIter) value() base.LazyValue {
-	return base.MakeInPlaceValue(i.val)
+	return i.lazyValue
 }
 
 // Error implements internalIterator.Error, as documented in the pebble
@@ -945,6 +1415,8 @@ func (i *blockIter) Close() error {
 	i.cacheHandle.Release()
 	i.cacheHandle = cache.Handle{}
 	i.val = nil
+	i.lazyValue = base.LazyValue{}
+	i.lazyValueHandling.vbr = nil
 	return nil
 }
 
@@ -966,7 +1438,7 @@ func (i *blockIter) valid() bool {
 // gathers all the fragments with identical bounds within a block and returns a
 // single keyspan.Span describing all the keys defined over the span.
 //
-// Memory lifetime
+// # Memory lifetime
 //
 // A Span returned by fragmentBlockIter is only guaranteed to be stable until
 // the next fragmentBlockIter iteration positioning method. A Span's Keys slice
@@ -1219,8 +1691,13 @@ func (i *fragmentBlockIter) Prev() *keyspan.Span {
 
 // SeekGE implements (keyspan.FragmentIterator).SeekGE.
 func (i *fragmentBlockIter) SeekGE(k []byte) *keyspan.Span {
-	i.dir = +1
-	return i.gatherForward(i.blockIter.SeekGE(k, base.SeekGEFlags(0)))
+	if s := i.SeekLT(k); s != nil && i.blockIter.cmp(k, s.End) < 0 {
+		return s
+	}
+	// TODO(jackson): If the above i.SeekLT(k) discovers a span but the span
+	// doesn't meet the k < s.End comparison, then there's no need for the
+	// SeekLT to gatherBackward.
+	return i.Next()
 }
 
 // SeekLT implements (keyspan.FragmentIterator).SeekLT.

vendor/github.com/cockroachdb/pebble/sstable/block_property.go

@@ -82,17 +82,18 @@ import (
 // properties must only apply to the key, and will be provided a nil value.
 
 // BlockPropertyCollector is used when writing a sstable.
-// - All calls to Add are included in the next FinishDataBlock, after which
-//   the next data block is expected to start.
 //
-// - The index entry generated for the data block, which contains the return
-//   value from FinishDataBlock, is not immediately included in the current
-//   index block. It is included when AddPrevDataBlockToIndexBlock is called.
-//   An alternative would be to return an opaque handle from FinishDataBlock
-//   and pass it to a new AddToIndexBlock method, which requires more
-//   plumbing, and passing of an interface{} results in a undesirable heap
-//   allocation. AddPrevDataBlockToIndexBlock must be called before keys are
-//   added to the new data block.
+//   - All calls to Add are included in the next FinishDataBlock, after which
+//     the next data block is expected to start.
+//
+//   - The index entry generated for the data block, which contains the return
+//     value from FinishDataBlock, is not immediately included in the current
+//     index block. It is included when AddPrevDataBlockToIndexBlock is called.
+//     An alternative would be to return an opaque handle from FinishDataBlock
+//     and pass it to a new AddToIndexBlock method, which requires more
+//     plumbing, and passing of an interface{} results in a undesirable heap
+//     allocation. AddPrevDataBlockToIndexBlock must be called before keys are
+//     added to the new data block.
 type BlockPropertyCollector interface {
 	// Name returns the name of the block property collector.
 	Name() string
@@ -586,9 +587,9 @@ var blockPropertiesFiltererPool = sync.Pool{
 	},
 }
 
-// NewBlockPropertiesFilterer returns a partially initialized filterer. To complete
+// newBlockPropertiesFilterer returns a partially initialized filterer. To complete
 // initialization, call IntersectsUserPropsAndFinishInit.
-func NewBlockPropertiesFilterer(
+func newBlockPropertiesFilterer(
 	filters []BlockPropertyFilter, limited BoundLimitedBlockPropertyFilter,
 ) *BlockPropertiesFilterer {
 	filterer := blockPropertiesFiltererPool.Get().(*BlockPropertiesFilterer)
@@ -608,11 +609,31 @@ func releaseBlockPropertiesFilterer(filterer *BlockPropertiesFilterer) {
 	blockPropertiesFiltererPool.Put(filterer)
 }
 
-// IntersectsUserPropsAndFinishInit is called with the user properties map for
+// IntersectsTable evaluates the provided block-property filter against the
+// provided set of table-level properties. If there is no intersection between
+// the filters and the table or an error is encountered, IntersectsTable returns
+// a nil filterer (and possibly an error). If there is an intersection,
+// IntersectsTable returns a non-nil filterer that may be used by an iterator
+// reading the table.
+func IntersectsTable(
+	filters []BlockPropertyFilter,
+	limited BoundLimitedBlockPropertyFilter,
+	userProperties map[string]string,
+) (*BlockPropertiesFilterer, error) {
+	f := newBlockPropertiesFilterer(filters, limited)
+	ok, err := f.intersectsUserPropsAndFinishInit(userProperties)
+	if !ok || err != nil {
+		releaseBlockPropertiesFilterer(f)
+		return nil, err
+	}
+	return f, nil
+}
+
+// intersectsUserPropsAndFinishInit is called with the user properties map for
 // the sstable and returns whether the sstable intersects the filters. It
 // additionally initializes the shortIDToFiltersIndex for the filters that are
 // relevant to this sstable.
-func (f *BlockPropertiesFilterer) IntersectsUserPropsAndFinishInit(
+func (f *BlockPropertiesFilterer) intersectsUserPropsAndFinishInit(
 	userProperties map[string]string,
 ) (bool, error) {
 	for i := range f.filters {

vendor/github.com/cockroachdb/pebble/sstable/filter.go

@@ -18,14 +18,32 @@ type FilterMetrics struct {
 	Misses int64
 }
 
-var dummyFilterMetrics FilterMetrics
+// FilterMetricsTracker is used to keep track of filter metrics. It contains the
+// same metrics as FilterMetrics, but they can be updated atomically. An
+// instance of FilterMetricsTracker can be passed to a Reader as a ReaderOption.
+type FilterMetricsTracker struct {
+	// See FilterMetrics.Hits.
+	hits atomic.Int64
+	// See FilterMetrics.Misses.
+	misses atomic.Int64
+}
 
-func (m *FilterMetrics) readerApply(r *Reader) {
+var _ ReaderOption = (*FilterMetricsTracker)(nil)
+
+func (m *FilterMetricsTracker) readerApply(r *Reader) {
 	if r.tableFilter != nil {
 		r.tableFilter.metrics = m
 	}
 }
 
+// Load returns the current values as FilterMetrics.
+func (m *FilterMetricsTracker) Load() FilterMetrics {
+	return FilterMetrics{
+		Hits:   m.hits.Load(),
+		Misses: m.misses.Load(),
+	}
+}
+
 // BlockHandle is the file offset and length of a block.
 type BlockHandle struct {
 	Offset, Length uint64
@@ -47,22 +65,24 @@ type filterWriter interface {
 
 type tableFilterReader struct {
 	policy  FilterPolicy
-	metrics *FilterMetrics
+	metrics *FilterMetricsTracker
 }
 
 func newTableFilterReader(policy FilterPolicy) *tableFilterReader {
 	return &tableFilterReader{
 		policy:  policy,
-		metrics: &dummyFilterMetrics,
+		metrics: nil,
 	}
 }
 
 func (f *tableFilterReader) mayContain(data, key []byte) bool {
 	mayContain := f.policy.MayContain(TableFilter, data, key)
-	if mayContain {
-		atomic.AddInt64(&f.metrics.Misses, 1)
-	} else {
-		atomic.AddInt64(&f.metrics.Hits, 1)
+	if f.metrics != nil {
+		if mayContain {
+			f.metrics.misses.Add(1)
+		} else {
+			f.metrics.hits.Add(1)
+		}
 	}
 	return mayContain
 }

vendor/github.com/cockroachdb/pebble/sstable/format.go

@@ -23,8 +23,10 @@ const (
 	TableFormatRocksDBv2
 	TableFormatPebblev1 // Block properties.
 	TableFormatPebblev2 // Range keys.
+	TableFormatPebblev3 // Value blocks.
+	TableFormatPebblev4 // Reserved.
 
-	TableFormatMax = TableFormatPebblev2
+	TableFormatMax = TableFormatPebblev4
 )
 
 // ParseTableFormat parses the given magic bytes and version into its
@@ -46,6 +48,10 @@ func ParseTableFormat(magic []byte, version uint32) (TableFormat, error) {
 			return TableFormatPebblev1, nil
 		case 2:
 			return TableFormatPebblev2, nil
+		case 3:
+			return TableFormatPebblev3, nil
+		case 4:
+			return TableFormatPebblev4, nil
 		default:
 			return TableFormatUnspecified, base.CorruptionErrorf(
 				"pebble/table: unsupported pebble format version %d", errors.Safe(version),
@@ -53,7 +59,7 @@ func ParseTableFormat(magic []byte, version uint32) (TableFormat, error) {
 		}
 	default:
 		return TableFormatUnspecified, base.CorruptionErrorf(
-			"pebble/table: invalid table (bad magic number)",
+			"pebble/table: invalid table (bad magic number: 0x%x)", magic,
 		)
 	}
 }
@@ -69,6 +75,10 @@ func (f TableFormat) AsTuple() (string, uint32) {
 		return pebbleDBMagic, 1
 	case TableFormatPebblev2:
 		return pebbleDBMagic, 2
+	case TableFormatPebblev3:
+		return pebbleDBMagic, 3
+	case TableFormatPebblev4:
+		return pebbleDBMagic, 4
 	default:
 		panic("sstable: unknown table format version tuple")
 	}
@@ -85,6 +95,10 @@ func (f TableFormat) String() string {
 		return "(Pebble,v1)"
 	case TableFormatPebblev2:
 		return "(Pebble,v2)"
+	case TableFormatPebblev3:
+		return "(Pebble,v3)"
+	case TableFormatPebblev4:
+		return "(Pebble,v4)"
 	default:
 		panic("sstable: unknown table format version tuple")
 	}

vendor/github.com/cockroachdb/pebble/sstable/internal.go

@@ -21,6 +21,7 @@ const (
 	InternalKeyKindSet             = base.InternalKeyKindSet
 	InternalKeyKindMerge           = base.InternalKeyKindMerge
 	InternalKeyKindLogData         = base.InternalKeyKindLogData
+	InternalKeyKindSingleDelete    = base.InternalKeyKindSingleDelete
 	InternalKeyKindRangeDelete     = base.InternalKeyKindRangeDelete
 	InternalKeyKindMax             = base.InternalKeyKindMax
 	InternalKeyKindInvalid         = base.InternalKeyKindInvalid

vendor/github.com/cockroachdb/pebble/sstable/options.go

@@ -117,6 +117,9 @@ type ReaderOptions struct {
 	// written with {Batch,DB}.Merge. The MergerName is checked for consistency
 	// with the value stored in the sstable when it was written.
 	MergerName string
+
+	// Logger is an optional logger and tracer.
+	LoggerAndTracer base.LoggerAndTracer
 }
 
 func (o ReaderOptions) ensureDefaults() ReaderOptions {
@@ -126,6 +129,9 @@ func (o ReaderOptions) ensureDefaults() ReaderOptions {
 	if o.MergerName == "" {
 		o.MergerName = base.DefaultMerger.Name
 	}
+	if o.LoggerAndTracer == nil {
+		o.LoggerAndTracer = base.NoopLoggerAndTracer{}
+	}
 	return o
 }
 
@@ -221,6 +227,14 @@ type WriterOptions struct {
 	// compress data blocks and write datablocks to disk in parallel with the
 	// Writer client goroutine.
 	Parallelism bool
+
+	// ShortAttributeExtractor mirrors
+	// Options.Experimental.ShortAttributeExtractor.
+	ShortAttributeExtractor base.ShortAttributeExtractor
+
+	// RequiredInPlaceValueBound mirrors
+	// Options.Experimental.RequiredInPlaceValueBound.
+	RequiredInPlaceValueBound UserKeyPrefixBound
 }
 
 func (o WriterOptions) ensureDefaults() WriterOptions {

vendor/github.com/cockroachdb/pebble/sstable/properties.go

@@ -120,6 +120,10 @@ type Properties struct {
 	NumRangeKeySets uint64 `prop:"pebble.num.range-key-sets"`
 	// The number of RANGEKEYUNSETs in this table.
 	NumRangeKeyUnsets uint64 `prop:"pebble.num.range-key-unsets"`
+	// The number of value blocks in this table. Only serialized if > 0.
+	NumValueBlocks uint64 `prop:"pebble.num.value-blocks"`
+	// The number of values stored in value blocks. Only serialized if > 0.
+	NumValuesInValueBlocks uint64 `prop:"pebble.num.values.in.value-blocks"`
 	// Timestamp of the earliest key. 0 if unknown.
 	OldestKeyTime uint64 `prop:"rocksdb.oldest.key.time"`
 	// The name of the prefix extractor used in this table. Empty if no prefix
@@ -132,16 +136,29 @@ type Properties struct {
 	PropertyCollectorNames string `prop:"rocksdb.property.collectors"`
 	// Total raw key size.
 	RawKeySize uint64 `prop:"rocksdb.raw.key.size"`
+	// Total raw key size of point deletion tombstones. This value is comparable
+	// to RawKeySize.
+	RawPointTombstoneKeySize uint64 `prop:"pebble.raw.point-tombstone.key.size"`
 	// Total raw rangekey key size.
 	RawRangeKeyKeySize uint64 `prop:"pebble.raw.range-key.key.size"`
 	// Total raw rangekey value size.
 	RawRangeKeyValueSize uint64 `prop:"pebble.raw.range-key.value.size"`
 	// Total raw value size.
 	RawValueSize uint64 `prop:"rocksdb.raw.value.size"`
+	// The total number of keys in this table that were pinned by open snapshots.
+	SnapshotPinnedKeys uint64 `prop:"pebble.num.snapshot-pinned-keys"`
+	// The cumulative bytes of keys in this table that were pinned by
+	// open snapshots. This value is comparable to RawKeySize.
+	SnapshotPinnedKeySize uint64 `prop:"pebble.raw.snapshot-pinned-keys.size"`
+	// The cumulative bytes of values in this table that were pinned by
+	// open snapshots. This value is comparable to RawValueSize.
+	SnapshotPinnedValueSize uint64 `prop:"pebble.raw.snapshot-pinned-values.size"`
 	// Size of the top-level index if kTwoLevelIndexSearch is used.
 	TopLevelIndexSize uint64 `prop:"rocksdb.top-level.index.size"`
 	// User collected properties.
 	UserProperties map[string]string
+	// Total size of value blocks and value index block. Only serialized if > 0.
+	ValueBlocksSize uint64 `prop:"pebble.value-blocks.size"`
 	// If filtering is enabled, was the filter created on the whole key.
 	WholeKeyFiltering bool `prop:"rocksdb.block.based.table.whole.key.filtering"`
 
@@ -283,7 +300,7 @@ func (p *Properties) saveString(m map[string][]byte, offset uintptr, value strin
 	m[propOffsetTagMap[offset]] = []byte(value)
 }
 
-func (p *Properties) save(w *rawBlockWriter) {
+func (p *Properties) save(tblFormat TableFormat, w *rawBlockWriter) {
 	m := make(map[string][]byte)
 	for k, v := range p.UserProperties {
 		m[k] = []byte(v)
@@ -333,6 +350,13 @@ func (p *Properties) save(w *rawBlockWriter) {
 	p.saveUvarint(m, unsafe.Offsetof(p.NumDeletions), p.NumDeletions)
 	p.saveUvarint(m, unsafe.Offsetof(p.NumMergeOperands), p.NumMergeOperands)
 	p.saveUvarint(m, unsafe.Offsetof(p.NumRangeDeletions), p.NumRangeDeletions)
+	// NB: We only write out the RawTombstoneKeySize for Pebble formats. This
+	// isn't strictly necessary because unrecognized properties are interpreted
+	// as user-defined properties, however writing them prevents byte-for-byte
+	// equivalence with RocksDB files that some of our testing requires.
+	if p.RawPointTombstoneKeySize > 0 && tblFormat >= TableFormatPebblev1 {
+		p.saveUvarint(m, unsafe.Offsetof(p.RawPointTombstoneKeySize), p.RawPointTombstoneKeySize)
+	}
 	if p.NumRangeKeys() > 0 {
 		p.saveUvarint(m, unsafe.Offsetof(p.NumRangeKeyDels), p.NumRangeKeyDels)
 		p.saveUvarint(m, unsafe.Offsetof(p.NumRangeKeySets), p.NumRangeKeySets)
@@ -340,6 +364,12 @@ func (p *Properties) save(w *rawBlockWriter) {
 		p.saveUvarint(m, unsafe.Offsetof(p.RawRangeKeyKeySize), p.RawRangeKeyKeySize)
 		p.saveUvarint(m, unsafe.Offsetof(p.RawRangeKeyValueSize), p.RawRangeKeyValueSize)
 	}
+	if p.NumValueBlocks > 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.NumValueBlocks), p.NumValueBlocks)
+	}
+	if p.NumValuesInValueBlocks > 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.NumValuesInValueBlocks), p.NumValuesInValueBlocks)
+	}
 	p.saveUvarint(m, unsafe.Offsetof(p.OldestKeyTime), p.OldestKeyTime)
 	if p.PrefixExtractorName != "" {
 		p.saveString(m, unsafe.Offsetof(p.PrefixExtractorName), p.PrefixExtractorName)
@@ -348,8 +378,16 @@ func (p *Properties) save(w *rawBlockWriter) {
 	if p.PropertyCollectorNames != "" {
 		p.saveString(m, unsafe.Offsetof(p.PropertyCollectorNames), p.PropertyCollectorNames)
 	}
+	if p.SnapshotPinnedKeys > 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.SnapshotPinnedKeys), p.SnapshotPinnedKeys)
+		p.saveUvarint(m, unsafe.Offsetof(p.SnapshotPinnedKeySize), p.SnapshotPinnedKeySize)
+		p.saveUvarint(m, unsafe.Offsetof(p.SnapshotPinnedValueSize), p.SnapshotPinnedValueSize)
+	}
 	p.saveUvarint(m, unsafe.Offsetof(p.RawKeySize), p.RawKeySize)
 	p.saveUvarint(m, unsafe.Offsetof(p.RawValueSize), p.RawValueSize)
+	if p.ValueBlocksSize > 0 {
+		p.saveUvarint(m, unsafe.Offsetof(p.ValueBlocksSize), p.ValueBlocksSize)
+	}
 	p.saveBool(m, unsafe.Offsetof(p.WholeKeyFiltering), p.WholeKeyFiltering)
 
 	keys := make([]string, 0, len(m))

vendor/github.com/cockroachdb/pebble/sstable/raw_block.go

@@ -26,7 +26,8 @@ func (w *rawBlockWriter) add(key InternalKey, value []byte) {
 	w.curKey = w.curKey[:size]
 	copy(w.curKey, key.UserKey)
 
-	w.store(size, value)
+	w.storeWithOptionalValuePrefix(
+		size, value, len(key.UserKey), false, 0, false)
 }
 
 // rawBlockIter is an iterator over a single block of data. Unlike blockIter,

vendor/github.com/cockroachdb/pebble/sstable/suffix_rewriter.go

@@ -2,23 +2,41 @@ package sstable
 
 import (
 	"bytes"
+	"context"
 	"math"
-	"os"
 	"sync"
-	"time"
 
 	"github.com/cespare/xxhash/v2"
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/bytealloc"
+	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/rangekey"
+	"github.com/cockroachdb/pebble/objstorage"
 )
 
-// RewriteKeySuffixes copies the content of the passed SSTable bytes to a new
-// sstable, written to `out`, in which the suffix `from` has is replaced with
-// `to` in every key. The input sstable must consist of only Sets or RangeKeySets
-// and every key must have `from` as its suffix as determined by the Split
-// function of the Comparer in the passed WriterOptions. Range deletes must not
-// exist in this sstable, as they will be ignored.
+// RewriteKeySuffixes is deprecated.
+//
+// TODO(sumeer): remove after switching CockroachDB to RewriteKeySuffixesAndReturnFormat.
+func RewriteKeySuffixes(
+	sst []byte,
+	rOpts ReaderOptions,
+	out objstorage.Writable,
+	o WriterOptions,
+	from, to []byte,
+	concurrency int,
+) (*WriterMetadata, error) {
+	meta, _, err := RewriteKeySuffixesAndReturnFormat(sst, rOpts, out, o, from, to, concurrency)
+	return meta, err
+}
+
+// RewriteKeySuffixesAndReturnFormat copies the content of the passed SSTable
+// bytes to a new sstable, written to `out`, in which the suffix `from` has is
+// replaced with `to` in every key. The input sstable must consist of only
+// Sets or RangeKeySets and every key must have `from` as its suffix as
+// determined by the Split function of the Comparer in the passed
+// WriterOptions. Range deletes must not exist in this sstable, as they will
+// be ignored.
 //
 // Data blocks are rewritten in parallel by `concurrency` workers and then
 // assembled into a final SST. Filters are copied from the original SST without
@@ -27,58 +45,78 @@ import (
 //
 // Any block and table property collectors configured in the WriterOptions must
 // implement SuffixReplaceableTableCollector/SuffixReplaceableBlockCollector.
-func RewriteKeySuffixes(
+//
+// The WriterOptions.TableFormat is ignored, and the output sstable has the
+// same TableFormat as the input, which is returned in case the caller wants
+// to do some error checking. Suffix rewriting is meant to be efficient, and
+// allowing changes in the TableFormat detracts from that efficiency.
+func RewriteKeySuffixesAndReturnFormat(
 	sst []byte,
 	rOpts ReaderOptions,
-	out writeCloseSyncer,
+	out objstorage.Writable,
 	o WriterOptions,
 	from, to []byte,
 	concurrency int,
-) (*WriterMetadata, error) {
+) (*WriterMetadata, TableFormat, error) {
 	r, err := NewMemReader(sst, rOpts)
 	if err != nil {
-		return nil, err
+		return nil, TableFormatUnspecified, err
 	}
 	defer r.Close()
 	return rewriteKeySuffixesInBlocks(r, out, o, from, to, concurrency)
 }
 
 func rewriteKeySuffixesInBlocks(
-	r *Reader, out writeCloseSyncer, o WriterOptions, from, to []byte, concurrency int,
-) (*WriterMetadata, error) {
+	r *Reader, out objstorage.Writable, o WriterOptions, from, to []byte, concurrency int,
+) (*WriterMetadata, TableFormat, error) {
 	if o.Comparer == nil || o.Comparer.Split == nil {
-		return nil, errors.New("a valid splitter is required to define suffix to replace replace suffix")
+		return nil, TableFormatUnspecified,
+			errors.New("a valid splitter is required to rewrite suffixes")
 	}
 	if concurrency < 1 {
-		return nil, errors.New("concurrency must be >= 1")
+		return nil, TableFormatUnspecified, errors.New("concurrency must be >= 1")
+	}
+	// Even though NumValueBlocks = 0 => NumValuesInValueBlocks = 0, check both
+	// as a defensive measure.
+	if r.Properties.NumValueBlocks > 0 || r.Properties.NumValuesInValueBlocks > 0 {
+		return nil, TableFormatUnspecified,
+			errors.New("sstable with a single suffix should not have value blocks")
 	}
 
+	tableFormat := r.tableFormat
+	o.TableFormat = tableFormat
 	w := NewWriter(out, o)
-	defer w.Close()
+	defer func() {
+		if w != nil {
+			w.Close()
+		}
+	}()
 
 	for _, c := range w.propCollectors {
 		if _, ok := c.(SuffixReplaceableTableCollector); !ok {
-			return nil, errors.Errorf("property collector %s does not support suffix replacement", c.Name())
+			return nil, TableFormatUnspecified,
+				errors.Errorf("property collector %s does not support suffix replacement", c.Name())
 		}
 	}
 	for _, c := range w.blockPropCollectors {
 		if _, ok := c.(SuffixReplaceableBlockCollector); !ok {
-			return nil, errors.Errorf("block property collector %s does not support suffix replacement", c.Name())
+			return nil, TableFormatUnspecified,
+				errors.Errorf("block property collector %s does not support suffix replacement", c.Name())
 		}
 	}
 
 	l, err := r.Layout()
 	if err != nil {
-		return nil, errors.Wrap(err, "reading layout")
+		return nil, TableFormatUnspecified, errors.Wrap(err, "reading layout")
 	}
 
 	if err := rewriteDataBlocksToWriter(r, w, l.Data, from, to, w.split, concurrency); err != nil {
-		return nil, errors.Wrap(err, "rewriting data blocks")
+		return nil, TableFormatUnspecified, errors.Wrap(err, "rewriting data blocks")
 	}
 
 	// Copy over the range key block and replace suffixes in it if it exists.
 	if err := rewriteRangeKeyBlockToWriter(r, w, from, to); err != nil {
-		return nil, errors.Wrap(err, "rewriting range key blocks")
+		return nil, TableFormatUnspecified, errors.Wrap(err, "rewriting range key blocks")
 	}
 
 	// Copy over the filter block if it exists (rewriteDataBlocksToWriter will
@@ -86,7 +124,7 @@ func rewriteKeySuffixesInBlocks(
 	if w.filter != nil && l.Filter.Length > 0 {
 		filterBlock, _, err := readBlockBuf(r, l.Filter, nil)
 		if err != nil {
-			return nil, errors.Wrap(err, "reading filter")
+			return nil, TableFormatUnspecified, errors.Wrap(err, "reading filter")
 		}
 		w.filter = copyFilterWriter{
 			origPolicyName: w.filter.policyName(), origMetaName: w.filter.metaName(), data: filterBlock,
@@ -94,10 +132,12 @@ func rewriteKeySuffixesInBlocks(
 	}
 
 	if err := w.Close(); err != nil {
-		return nil, err
+		w = nil
+		return nil, TableFormatUnspecified, err
 	}
-
-	return w.Metadata()
+	writerMeta, err := w.Metadata()
+	w = nil
+	return writerMeta, tableFormat, err
 }
 
 var errBadKind = errors.New("key does not have expected kind (set)")
@@ -126,8 +166,8 @@ func rewriteBlocks(
 		buf.checksummer.xxHasher = xxhash.New()
 	}
 
-	var blockAlloc []byte
-	var keyAlloc []byte
+	var blockAlloc bytealloc.A
+	var keyAlloc bytealloc.A
 	var scratch InternalKey
 
 	var inputBlock, inputBlockBuf []byte
@@ -178,7 +218,24 @@ func rewriteBlocks(
 			copy(scratch.UserKey, key.UserKey[:si])
 			copy(scratch.UserKey[si:], to)
 
+			// NB: for TableFormatPebblev3 and higher, since
+			// !iter.lazyValueHandling.hasValuePrefix, it will return the raw value
+			// in the block, which includes the 1-byte prefix. This is fine since bw
+			// also does not know about the prefix and will preserve it in bw.add.
 			v := val.InPlaceValue()
+			if invariants.Enabled && r.tableFormat >= TableFormatPebblev3 &&
+				key.Kind() == InternalKeyKindSet {
+				if len(v) < 1 {
+					return errors.Errorf("value has no prefix")
+				}
+				prefix := valuePrefix(v[0])
+				if isValueHandle(prefix) {
+					return errors.Errorf("value prefix is incorrect")
+				}
+				if setHasSamePrefix(prefix) {
+					return errors.Errorf("multiple keys with same key prefix")
+				}
+			}
 			bw.add(scratch, v)
 			if output[i].start.UserKey == nil {
 				keyAlloc, output[i].start = cloneKeyWithBuf(scratch, keyAlloc)
@@ -191,12 +248,7 @@ func rewriteBlocks(
 		finished := compressAndChecksum(bw.finish(), compression, &buf)
 
 		// copy our finished block into the output buffer.
-		sz := len(finished) + blockTrailerLen
-		if cap(blockAlloc) < sz {
-			blockAlloc = make([]byte, sz*128)
-		}
-		output[i].data = blockAlloc[:sz:sz]
-		blockAlloc = blockAlloc[sz:]
+		blockAlloc, output[i].data = blockAlloc.Alloc(len(finished) + blockTrailerLen)
 		copy(output[i].data, finished)
 		copy(output[i].data[len(finished):], buf.tmp[:blockTrailerLen])
 	}
@@ -278,11 +330,11 @@ func rewriteDataBlocksToWriter(
 
 	for i := range blocks {
 		// Write the rewritten block to the file.
-		n, err := w.writer.Write(blocks[i].data)
-		if err != nil {
+		if err := w.writable.Write(blocks[i].data); err != nil {
 			return err
 		}
 
+		n := len(blocks[i].data)
 		bh := BlockHandle{Offset: w.meta.Size, Length: uint64(n) - blockTrailerLen}
 		// Update the overall size.
 		w.meta.Size += uint64(n)
@@ -306,8 +358,8 @@ func rewriteDataBlocksToWriter(
 			}
 		}
 
-		var bhp BlockHandleWithProperties
-		if bhp, err = w.maybeAddBlockPropertiesToBlockHandle(bh); err != nil {
+		bhp, err := w.maybeAddBlockPropertiesToBlockHandle(bh)
+		if err != nil {
 			return err
 		}
 		var nextKey InternalKey
@@ -384,13 +436,18 @@ func (c copyFilterWriter) policyName() string      { return c.origPolicyName }
 // more work to rederive filters, props, etc, however re-doing that work makes
 // it less restrictive -- props no longer need to
 func RewriteKeySuffixesViaWriter(
-	r *Reader, out writeCloseSyncer, o WriterOptions, from, to []byte,
+	r *Reader, out objstorage.Writable, o WriterOptions, from, to []byte,
 ) (*WriterMetadata, error) {
 	if o.Comparer == nil || o.Comparer.Split == nil {
-		return nil, errors.New("a valid splitter is required to define suffix to replace replace suffix")
+		return nil, errors.New("a valid splitter is required to rewrite suffixes")
 	}
 
 	w := NewWriter(out, o)
+	defer func() {
+		if w != nil {
+			w.Close()
+		}
+	}()
 	i, err := r.NewIter(nil, nil)
 	if err != nil {
 		return nil, err
@@ -424,18 +481,21 @@ func RewriteKeySuffixesViaWriter(
 		return nil, err
 	}
 	if err := w.Close(); err != nil {
+		w = nil
 		return nil, err
 	}
-	return &w.meta, nil
+	writerMeta, err := w.Metadata()
+	w = nil
+	return writerMeta, err
 }
 
 // NewMemReader opens a reader over the SST stored in the passed []byte.
 func NewMemReader(sst []byte, o ReaderOptions) (*Reader, error) {
-	return NewReader(memReader{sst, bytes.NewReader(sst), sizeOnlyStat(int64(len(sst)))}, o)
+	return NewReader(newMemReader(sst), o)
 }
 
 func readBlockBuf(r *Reader, bh BlockHandle, buf []byte) ([]byte, []byte, error) {
-	raw := r.file.(memReader).b[bh.Offset : bh.Offset+bh.Length+blockTrailerLen]
+	raw := r.readable.(*memReader).b[bh.Offset : bh.Offset+bh.Length+blockTrailerLen]
 	if err := checkChecksum(r.checksumType, raw, bh, 0); err != nil {
 		return nil, buf, err
 	}
@@ -455,31 +515,46 @@ func readBlockBuf(r *Reader, bh BlockHandle, buf []byte) ([]byte, []byte, error)
 	return res, buf, err
 }
 
-// memReader is a thin wrapper around a []byte such that it can be passed to an
+// memReader is a thin wrapper around a []byte such that it can be passed to
 // sstable.Reader. It supports concurrent use, and does so without locking in
 // contrast to the heavier read/write vfs.MemFile.
 type memReader struct {
-	b []byte
-	r *bytes.Reader
-	s sizeOnlyStat
+	b  []byte
+	r  *bytes.Reader
+	rh objstorage.NoopReadHandle
 }
 
-var _ ReadableFile = memReader{}
+var _ objstorage.Readable = (*memReader)(nil)
 
-// ReadAt implements io.ReaderAt.
-func (m memReader) ReadAt(p []byte, off int64) (n int, err error) { return m.r.ReadAt(p, off) }
+func newMemReader(b []byte) *memReader {
+	r := &memReader{
+		b: b,
+		r: bytes.NewReader(b),
+	}
+	r.rh = objstorage.MakeNoopReadHandle(r)
+	return r
+}
 
-// Close implements io.Closer.
-func (memReader) Close() error { return nil }
+// ReadAt is part of objstorage.Readable.
+func (m *memReader) ReadAt(_ context.Context, p []byte, off int64) error {
+	n, err := m.r.ReadAt(p, off)
+	if invariants.Enabled && err == nil && n != len(p) {
+		panic("short read")
+	}
+	return err
+}
 
-// Stat implements ReadableFile.
-func (m memReader) Stat() (os.FileInfo, error) { return m.s, nil }
+// Close is part of objstorage.Readable.
+func (*memReader) Close() error {
+	return nil
+}
 
-type sizeOnlyStat int64
+// Stat is part of objstorage.Readable.
+func (m *memReader) Size() int64 {
+	return int64(len(m.b))
+}
 
-func (s sizeOnlyStat) Size() int64      { return int64(s) }
-func (sizeOnlyStat) IsDir() bool        { panic(errors.AssertionFailedf("unimplemented")) }
-func (sizeOnlyStat) ModTime() time.Time { panic(errors.AssertionFailedf("unimplemented")) }
-func (sizeOnlyStat) Mode() os.FileMode  { panic(errors.AssertionFailedf("unimplemented")) }
-func (sizeOnlyStat) Name() string       { panic(errors.AssertionFailedf("unimplemented")) }
-func (sizeOnlyStat) Sys() interface{}   { panic(errors.AssertionFailedf("unimplemented")) }
+// NewReadHandle is part of objstorage.Readable.
+func (m *memReader) NewReadHandle(_ context.Context) objstorage.ReadHandle {
+	return &m.rh
+}

vendor/github.com/cockroachdb/pebble/sstable/table.go

@@ -27,52 +27,53 @@
 //
 // To return the value for a key:
 //
-// 	r := table.NewReader(file, options)
-// 	defer r.Close()
-// 	i := r.NewIter(nil, nil)
-// 	defer i.Close()
-// 	ikey, value := r.SeekGE(key)
-// 	if options.Comparer.Compare(ikey.UserKey, key) != 0 {
-// 	  // not found
-// 	} else {
-// 	  // value is the first record containing key
-// 	}
+//	r := table.NewReader(file, options)
+//	defer r.Close()
+//	i := r.NewIter(nil, nil)
+//	defer i.Close()
+//	ikey, value := r.SeekGE(key)
+//	if options.Comparer.Compare(ikey.UserKey, key) != 0 {
+//	  // not found
+//	} else {
+//	  // value is the first record containing key
+//	}
 //
 // To count the number of entries in a table:
 //
-// 	i, n := r.NewIter(nil, nil), 0
-// 	for key, value := i.First(); key != nil; key, value = i.Next() {
-// 		n++
-// 	}
-// 	if err := i.Close(); err != nil {
-// 		return 0, err
-// 	}
-// 	return n, nil
+//	i, n := r.NewIter(nil, nil), 0
+//	for key, value := i.First(); key != nil; key, value = i.Next() {
+//		n++
+//	}
+//	if err := i.Close(); err != nil {
+//		return 0, err
+//	}
+//	return n, nil
 //
 // To write a table with three entries:
 //
-// 	w := table.NewWriter(file, options)
-// 	if err := w.Set([]byte("apple"), []byte("red")); err != nil {
-// 		w.Close()
-// 		return err
-// 	}
-// 	if err := w.Set([]byte("banana"), []byte("yellow")); err != nil {
-// 		w.Close()
-// 		return err
-// 	}
-// 	if err := w.Set([]byte("cherry"), []byte("red")); err != nil {
-// 		w.Close()
-// 		return err
-// 	}
-// 	return w.Close()
+//	w := table.NewWriter(file, options)
+//	if err := w.Set([]byte("apple"), []byte("red")); err != nil {
+//		w.Close()
+//		return err
+//	}
+//	if err := w.Set([]byte("banana"), []byte("yellow")); err != nil {
+//		w.Close()
+//		return err
+//	}
+//	if err := w.Set([]byte("cherry"), []byte("red")); err != nil {
+//		w.Close()
+//		return err
+//	}
+//	return w.Close()
 package sstable // import "github.com/cockroachdb/pebble/sstable"
 
 import (
+	"context"
 	"encoding/binary"
-	"io"
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
 )
 
 /*
@@ -87,6 +88,9 @@ The table file format looks like:
 [index block] (for single level index)
 [meta rangedel block] (optional)
 [meta range key block] (optional)
+[value block 0] (optional)
+[value block M-1] (optional)
+[meta value index block] (optional)
 [meta properties block]
 [metaindex block]
 [footer]
@@ -110,8 +114,9 @@ byte of the trailer (i.e. the block type), and is serialized as little-endian.
 The block type gives the per-block compression used; each block is compressed
 independently. The checksum algorithm is described in the pebble/crc package.
 
-Most blocks, other than the meta filter block, contain key/value pairs. The
-remainder of this comment refers to the decompressed block, which has its 5 byte
+Most blocks, other than the meta filter block, value blocks and meta value
+index block, contain key/value pairs. The remainder of this comment refers to
+the decompressed block, containing key/value pairs, which has its 5 byte
 trailer stripped. The decompressed block data consists of a sequence of such
 key/value entries followed by a block suffix. Each key is encoded as a shared
 prefix length and a remainder string. For example, if two adjacent keys are
@@ -154,6 +159,9 @@ lower-level index blocks.
 The metaindex block also contains block handles as values, with keys being
 the names of the meta blocks.
 
+For a description of value blocks and the meta value index block, see
+value_block.go
+
 */
 
 const (
@@ -183,6 +191,7 @@ const (
 	rocksDBFormatVersion2 = 2
 
 	metaRangeKeyName   = "pebble.range_key"
+	metaValueIndexName = "pebble.value_index"
 	metaPropertiesName = "rocksdb.properties"
 	metaRangeDelName   = "rocksdb.range_del"
 	metaRangeDelV2Name = "rocksdb.range_del2"
@@ -274,17 +283,20 @@ func (t blockType) String() string {
 }
 
 // legacy (LevelDB) footer format:
-//    metaindex handle (varint64 offset, varint64 size)
-//    index handle     (varint64 offset, varint64 size)
-//    <padding> to make the total size 2 * BlockHandle::kMaxEncodedLength
-//    table_magic_number (8 bytes)
+//
+//	metaindex handle (varint64 offset, varint64 size)
+//	index handle     (varint64 offset, varint64 size)
+//	<padding> to make the total size 2 * BlockHandle::kMaxEncodedLength
+//	table_magic_number (8 bytes)
+//
 // new (RocksDB) footer format:
-//    checksum type (char, 1 byte)
-//    metaindex handle (varint64 offset, varint64 size)
-//    index handle     (varint64 offset, varint64 size)
-//    <padding> to make the total size 2 * BlockHandle::kMaxEncodedLength + 1
-//    footer version (4 bytes)
-//    table_magic_number (8 bytes)
+//
+//	checksum type (char, 1 byte)
+//	metaindex handle (varint64 offset, varint64 size)
+//	index handle     (varint64 offset, varint64 size)
+//	<padding> to make the total size 2 * BlockHandle::kMaxEncodedLength + 1
+//	footer version (4 bytes)
+//	table_magic_number (8 bytes)
 type footer struct {
 	format      TableFormat
 	checksum    ChecksumType
@@ -293,26 +305,22 @@ type footer struct {
 	footerBH    BlockHandle
 }
 
-func readFooter(f ReadableFile) (footer, error) {
+func readFooter(f objstorage.Readable) (footer, error) {
 	var footer footer
-	stat, err := f.Stat()
-	if err != nil {
-		return footer, errors.Wrap(err, "pebble/table: invalid table (could not stat file)")
-	}
-	if stat.Size() < minFooterLen {
+	size := f.Size()
+	if size < minFooterLen {
 		return footer, base.CorruptionErrorf("pebble/table: invalid table (file size is too small)")
 	}
 
 	buf := make([]byte, maxFooterLen)
-	off := stat.Size() - maxFooterLen
+	off := size - maxFooterLen
 	if off < 0 {
 		off = 0
+		buf = buf[:size]
 	}
-	n, err := f.ReadAt(buf, off)
-	if err != nil && err != io.EOF {
+	if err := f.ReadAt(context.TODO(), buf, off); err != nil {
 		return footer, errors.Wrap(err, "pebble/table: invalid table (could not read footer)")
 	}
-	buf = buf[:n]
 
 	switch magic := buf[len(buf)-len(rocksDBMagic):]; string(magic) {
 	case levelDBMagic:
@@ -354,11 +362,11 @@ func readFooter(f ReadableFile) (footer, error) {
 		buf = buf[1:]
 
 	default:
-		return footer, base.CorruptionErrorf("pebble/table: invalid table (bad magic number)")
+		return footer, base.CorruptionErrorf("pebble/table: invalid table (bad magic number: 0x%x)", magic)
 	}
 
 	{
-		end := uint64(stat.Size())
+		end := uint64(size)
 		var n int
 		footer.metaindexBH, n = decodeBlockHandle(buf)
 		if n == 0 || footer.metaindexBH.Offset+footer.metaindexBH.Length > end {
@@ -420,7 +428,7 @@ func supportsTwoLevelIndex(format TableFormat) bool {
 	switch format {
 	case TableFormatLevelDB:
 		return false
-	case TableFormatRocksDBv2, TableFormatPebblev1, TableFormatPebblev2:
+	case TableFormatRocksDBv2, TableFormatPebblev1, TableFormatPebblev2, TableFormatPebblev3, TableFormatPebblev4:
 		return true
 	default:
 		panic("sstable: unspecified table format version")

vendor/github.com/cockroachdb/pebble/sstable/value_block.go

@@ -0,0 +1,948 @@
+// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sstable
+
+import (
+	"context"
+	"encoding/binary"
+	"io"
+	"sync"
+	"unsafe"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/cache"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
+	"golang.org/x/exp/rand"
+)
+
+// Value blocks are supported in TableFormatPebblev3.
+//
+// 1. Motivation and overview
+//
+// Value blocks are a mechanism designed for sstables storing MVCC data, where
+// there can be many versions of a key that need to be kept, but only the
+// latest value is typically read (see the documentation for Comparer.Split
+// regarding MVCC keys). The goal is faster reads. Unlike Pebble versions,
+// which can be eagerly thrown away (except when there are snapshots), MVCC
+// versions are long-lived (e.g. default CockroachDB garbage collection
+// threshold for older versions is 24 hours) and can significantly slow down
+// reads. We have seen CockroachDB production workloads with very slow reads
+// due to:
+// - 100s of versions for each key in a table.
+//
+// - Tables with mostly MVCC garbage consisting of 2 versions per key -- a
+//   real key-value pair, followed by a key-value pair whose value (usually
+//   with zero byte length) indicates it is an MVCC tombstone.
+//
+// The value blocks mechanism attempts to improve read throughput in these
+// cases when the key size is smaller than the value sizes of older versions.
+// This is done by moving the value of an older version to a value block in a
+// different part of the sstable. This improves spatial locality of the data
+// being read by the workload, which increases caching effectiveness.
+//
+// Additionally, even when the key size is not smaller than the value of older
+// versions (e.g. secondary indexes in CockroachDB), TableFormatPebblev3
+// stores the result of key comparisons done at write time inside the sstable,
+// which makes stepping from one key prefix to the next prefix (i.e., skipping
+// over older versions of a MVCC key) more efficient by avoiding key
+// comparisons and key decoding. See the results in
+// https://github.com/cockroachdb/pebble/pull/2149 and more details in the
+// comment inside BenchmarkIteratorScanNextPrefix. These improvements are also
+// visible in end-to-end CockroachDB tests, as outlined in
+// https://github.com/cockroachdb/cockroach/pull/96652.
+//
+// In TableFormatPebblev3, each SET has a one byte value prefix that tells us
+// whether the value is in-place or in a value block. This 1 byte prefix
+// encodes additional information:
+//
+// - ShortAttribute: This is an attribute of the value. Currently, CockroachDB
+//   uses it to represent whether the value is a tombstone or not. This avoids
+//   the need to fetch a value from the value block if the caller only wants
+//   to figure out whether it is an MVCC tombstone. The length of the value is
+//   another attribute that the caller can be interested in, and it is also
+//   accessible without reading the value in the value block (see the value
+//   handle in the details section).
+//
+// - SET-same-prefix: this enables the aforementioned optimization when
+//   stepping from one key prefix to the next key prefix.
+//
+// We further optimize this iteration over prefixes by using the restart
+// points in a block to encode whether the SET at a restart point has the same
+// prefix since the last restart point. This allows us to skip over restart
+// points within the same block. See the comment in blockWriter, and how both
+// SET-same-prefix and the restart point information is used in
+// blockIter.nextPrefixV3.
+//
+// This flexibility of values that are in-place or in value blocks requires
+// flexibility in the iterator interface. The InternalIterator interface
+// returns a LazyValue instead of a byte slice. Additionally, pebble.Iterator
+// allows the caller to ask for a LazyValue. See lazy_value.go for details,
+// including the memory lifetime management.
+//
+// For historical discussions about this feature, see the issue
+// https://github.com/cockroachdb/pebble/issues/1170 and the prototype in
+// https://github.com/cockroachdb/pebble/pull/1443.
+//
+// The code in this file mainly covers value block and related encodings. We
+// discuss these in the next section.
+//
+// 2. Details
+//
+// Note that the notion of the latest value is local to the sstable. It is
+// possible that that latest value has been deleted by a sstable in a higher
+// level, and what is the latest value from the perspective of the whole LSM
+// is an older MVCC version. This only affects performance and not
+// correctness. This local knowledge is also why we continue to store these
+// older versions in the same sstable -- we need to be able to conveniently
+// read them. The code in this file is agnostic to the policy regarding what
+// should be stored in value blocks -- it allows even the latest MVCC version
+// to be stored in a value block. The policy decision in made in the
+// sstable.Writer. See Writer.makeAddPointDecisionV3.
+//
+// Data blocks contain two kinds of SET keys: those with in-place values and
+// those with a value handle. To distinguish these two cases we use a single
+// byte prefix (valuePrefix). This single byte prefix is split into multiple
+// parts, where nb represents information that is encoded in n bits.
+//
+// +---------------+--------------------+-----------+--------------------+
+// | value-kind 2b | SET-same-prefix 1b | unused 2b | short-attribute 3b |
+// +---------------+--------------------+-----------+--------------------+
+//
+// The 2 bit value-kind specifies whether this is an in-place value or a value
+// handle pointing to a value block. We use 2 bits here for future
+// representation of values that are in separate files. The 1 bit
+// SET-same-prefix is true if this key is a SET and is immediately preceded by
+// a SET that shares the same prefix. The 3 bit short-attribute is described
+// in base.ShortAttribute -- it stores user-defined attributes about the
+// value. It is unused for in-place values.
+//
+// Value Handle and Value Blocks:
+// valueHandles refer to values in value blocks. Value blocks are simpler than
+// normal data blocks (that contain key-value pairs, and allow for binary
+// search), which makes them cheap for value retrieval purposes. A valueHandle
+// is a tuple (valueLen, blockNum, offsetInBlock), where blockNum is the 0
+// indexed value block number and offsetInBlock is the byte offset in that
+// block containing the value. The valueHandle.valueLen is included since
+// there are multiple use cases in CockroachDB that need the value length but
+// not the value, for which we can avoid reading the value in the value block
+// (see
+// https://github.com/cockroachdb/pebble/issues/1170#issuecomment-958203245).
+//
+// A value block has a checksum like other blocks, and is optionally
+// compressed. An uncompressed value block is a sequence of values with no
+// separator or length (we rely on the valueHandle to demarcate). The
+// valueHandle.offsetInBlock points to the value, of length
+// valueHandle.valueLen. While writing a sstable, all the (possibly
+// compressed) value blocks need to be held in-memory until they can be
+// written. Value blocks are placed after the "meta rangedel" and "meta range
+// key" blocks since value blocks are considered less likely to be read.
+//
+// Meta Value Index Block:
+// Since the (key, valueHandle) pair are written before there is any knowledge
+// of the byte offset of the value block in the file, or its compressed
+// length, we need another lookup to map the valueHandle.blockNum to the
+// information needed to read it from the file. This information is provided
+// by the "value index block". The "value index block" is referred to by the
+// metaindex block. The design intentionally avoids making the "value index
+// block" a general purpose key-value block, since each caller wants to lookup
+// the information for a particular blockNum (there is no need for SeekGE
+// etc.). Instead, this index block stores a sequence of (blockNum,
+// blockOffset, blockLength) tuples, where the blockNums are consecutive
+// integers, and the tuples are encoded with a fixed width encoding. This
+// allows a reader to find the tuple for block K by looking at the offset
+// K*fixed-width. The fixed width for each field is decided by looking at the
+// maximum value of each of these fields. As a concrete example of a large
+// sstable with many value blocks, we constructed a 100MB sstable with many
+// versions and had 2475 value blocks (~32KB each). This sstable had this
+// tuple encoded using 2+4+2=8 bytes, which means the uncompressed value index
+// block was 2475*8=~19KB, which is modest. Therefore, we don't support more
+// than one value index block. Consider the example of 2 byte blockNum, 4 byte
+// blockOffset and 2 byte blockLen. The value index block will look like:
+//
+//   +---------------+------------------+---------------+
+//   | blockNum (2B) | blockOffset (4B) | blockLen (2B) |
+//   +---------------+------------------+---------------+
+//   |       0       |    7,123,456     |  30,000       |
+//   +---------------+------------------+---------------+
+//   |       1       |    7,153,456     |  20,000       |
+//   +---------------+------------------+---------------+
+//   |       2       |    7,173,456     |  25,567       |
+//   +---------------+------------------+---------------+
+//   |     ....      |      ...         |    ...        |
+//
+//
+// The metaindex block contains the valueBlocksIndexHandle which in addition
+// to the BlockHandle also specifies the widths of these tuple fields. In the
+// above example, the
+// valueBlockIndexHandle.{blockNumByteLength,blockOffsetByteLength,blockLengthByteLength}
+// will be (2,4,2).
+
+// valueHandle is stored with a key when the value is in a value block. This
+// handle is the pointer to that value.
+type valueHandle struct {
+	valueLen      uint32
+	blockNum      uint32
+	offsetInBlock uint32
+}
+
+// valuePrefix is the single byte prefix for either the in-place value or the
+// encoded valueHandle. It encoded multiple kinds of information.
+type valuePrefix byte
+
+const (
+	// 2 most-significant bits of valuePrefix encodes the value-kind.
+	valueKindMask           valuePrefix = '\xC0'
+	valueKindIsValueHandle  valuePrefix = '\x80'
+	valueKindIsInPlaceValue valuePrefix = '\x00'
+
+	// 1 bit indicates SET has same key prefix as immediately preceding key that
+	// is also a SET. If the immediately preceding key in the same block is a
+	// SET, AND this bit is 0, the prefix must have changed.
+	//
+	// Note that the current policy of only storing older MVCC versions in value
+	// blocks means that valueKindIsValueHandle => SET has same prefix. But no
+	// code should rely on this behavior. Also, SET has same prefix does *not*
+	// imply valueKindIsValueHandle.
+	setHasSameKeyPrefixMask valuePrefix = '\x20'
+
+	// 3 least-significant bits for the user-defined base.ShortAttribute.
+	// Undefined for valueKindIsInPlaceValue.
+	userDefinedShortAttributeMask valuePrefix = '\x07'
+)
+
+// valueHandle fields are varint encoded, so maximum 5 bytes each, plus 1 byte
+// for the valuePrefix. This could alternatively be group varint encoded, but
+// experiments were inconclusive
+// (https://github.com/cockroachdb/pebble/pull/1443#issuecomment-1270298802).
+const valueHandleMaxLen = 5*3 + 1
+
+// Assert blockHandleLikelyMaxLen >= valueHandleMaxLen.
+const _ = uint(blockHandleLikelyMaxLen - valueHandleMaxLen)
+
+func encodeValueHandle(dst []byte, v valueHandle) int {
+	n := 0
+	n += binary.PutUvarint(dst[n:], uint64(v.valueLen))
+	n += binary.PutUvarint(dst[n:], uint64(v.blockNum))
+	n += binary.PutUvarint(dst[n:], uint64(v.offsetInBlock))
+	return n
+}
+
+func makePrefixForValueHandle(setHasSameKeyPrefix bool, attribute base.ShortAttribute) valuePrefix {
+	prefix := valueKindIsValueHandle | valuePrefix(attribute)
+	if setHasSameKeyPrefix {
+		prefix = prefix | setHasSameKeyPrefixMask
+	}
+	return prefix
+}
+
+func makePrefixForInPlaceValue(setHasSameKeyPrefix bool) valuePrefix {
+	prefix := valueKindIsInPlaceValue
+	if setHasSameKeyPrefix {
+		prefix = prefix | setHasSameKeyPrefixMask
+	}
+	return prefix
+}
+
+func isValueHandle(b valuePrefix) bool {
+	return b&valueKindMask == valueKindIsValueHandle
+}
+
+// REQUIRES: isValueHandle(b)
+func getShortAttribute(b valuePrefix) base.ShortAttribute {
+	return base.ShortAttribute(b & userDefinedShortAttributeMask)
+}
+
+func setHasSamePrefix(b valuePrefix) bool {
+	return b&setHasSameKeyPrefixMask == setHasSameKeyPrefixMask
+}
+
+func decodeLenFromValueHandle(src []byte) (uint32, []byte) {
+	ptr := unsafe.Pointer(&src[0])
+	var v uint32
+	if a := *((*uint8)(ptr)); a < 128 {
+		v = uint32(a)
+		src = src[1:]
+	} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
+		v = uint32(b)<<7 | uint32(a)
+		src = src[2:]
+	} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
+		v = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+		src = src[3:]
+	} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
+		v = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+		src = src[4:]
+	} else {
+		d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
+		v = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+		src = src[5:]
+	}
+	return v, src
+}
+
+func decodeRemainingValueHandle(src []byte) valueHandle {
+	var vh valueHandle
+	ptr := unsafe.Pointer(&src[0])
+	// Manually inlined uvarint decoding. Saves ~25% in benchmarks. Unrolling
+	// a loop for i:=0; i<2; i++, saves ~6%.
+	var v uint32
+	if a := *((*uint8)(ptr)); a < 128 {
+		v = uint32(a)
+		ptr = unsafe.Pointer(uintptr(ptr) + 1)
+	} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
+		v = uint32(b)<<7 | uint32(a)
+		ptr = unsafe.Pointer(uintptr(ptr) + 2)
+	} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
+		v = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+		ptr = unsafe.Pointer(uintptr(ptr) + 3)
+	} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
+		v = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+		ptr = unsafe.Pointer(uintptr(ptr) + 4)
+	} else {
+		d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
+		v = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+		ptr = unsafe.Pointer(uintptr(ptr) + 5)
+	}
+	vh.blockNum = v
+
+	if a := *((*uint8)(ptr)); a < 128 {
+		v = uint32(a)
+	} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
+		v = uint32(b)<<7 | uint32(a)
+	} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
+		v = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+	} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
+		v = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+	} else {
+		d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
+		v = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+	}
+	vh.offsetInBlock = v
+
+	return vh
+}
+
+func decodeValueHandle(src []byte) valueHandle {
+	valLen, src := decodeLenFromValueHandle(src)
+	vh := decodeRemainingValueHandle(src)
+	vh.valueLen = valLen
+	return vh
+}
+
+// valueBlocksIndexHandle is placed in the metaindex if there are any value
+// blocks. If there are no value blocks, there is no value blocks index, and
+// no entry in the metaindex. Note that the lack of entry in the metaindex
+// should not be used to ascertain whether the values are prefixed, since the
+// former is an emergent property of the data that was written and not known
+// until all the key-value pairs in the sstable are written.
+type valueBlocksIndexHandle struct {
+	h                     BlockHandle
+	blockNumByteLength    uint8
+	blockOffsetByteLength uint8
+	blockLengthByteLength uint8
+}
+
+const valueBlocksIndexHandleMaxLen = blockHandleMaxLenWithoutProperties + 3
+
+// Assert blockHandleLikelyMaxLen >= valueBlocksIndexHandleMaxLen.
+const _ = uint(blockHandleLikelyMaxLen - valueBlocksIndexHandleMaxLen)
+
+func encodeValueBlocksIndexHandle(dst []byte, v valueBlocksIndexHandle) int {
+	n := encodeBlockHandle(dst, v.h)
+	dst[n] = v.blockNumByteLength
+	n++
+	dst[n] = v.blockOffsetByteLength
+	n++
+	dst[n] = v.blockLengthByteLength
+	n++
+	return n
+}
+
+func decodeValueBlocksIndexHandle(src []byte) (valueBlocksIndexHandle, int, error) {
+	var vbih valueBlocksIndexHandle
+	var n int
+	vbih.h, n = decodeBlockHandle(src)
+	if n <= 0 {
+		return vbih, 0, errors.Errorf("bad BlockHandle %x", src)
+	}
+	if len(src) != n+3 {
+		return vbih, 0, errors.Errorf("bad BlockHandle %x", src)
+	}
+	vbih.blockNumByteLength = src[n]
+	vbih.blockOffsetByteLength = src[n+1]
+	vbih.blockLengthByteLength = src[n+2]
+	return vbih, n + 3, nil
+}
+
+type valueBlocksAndIndexStats struct {
+	numValueBlocks         uint64
+	numValuesInValueBlocks uint64
+	// Includes both value blocks and value index block.
+	valueBlocksAndIndexSize uint64
+}
+
+// valueBlockWriter writes a sequence of value blocks, and the value blocks
+// index, for a sstable.
+type valueBlockWriter struct {
+	// The configured uncompressed block size and size threshold
+	blockSize, blockSizeThreshold int
+	// Configured compression.
+	compression Compression
+	// checksummer with configured checksum type.
+	checksummer checksummer
+	// Block finished callback.
+	blockFinishedFunc func(compressedSize int)
+
+	// buf is the current block being written to (uncompressed).
+	buf *blockBuffer
+	// compressedBuf is used for compressing the block.
+	compressedBuf *blockBuffer
+	// Sequence of blocks that are finished.
+	blocks []blockAndHandle
+	// Cumulative value block bytes written so far.
+	totalBlockBytes uint64
+	numValues       uint64
+}
+
+type blockAndHandle struct {
+	block      *blockBuffer
+	handle     BlockHandle
+	compressed bool
+}
+
+type blockBuffer struct {
+	b []byte
+}
+
+// Pool of block buffers that should be roughly the blockSize.
+var uncompressedValueBlockBufPool = sync.Pool{
+	New: func() interface{} {
+		return &blockBuffer{}
+	},
+}
+
+// Pool of block buffers for compressed value blocks. These may widely vary in
+// size based on compression ratios.
+var compressedValueBlockBufPool = sync.Pool{
+	New: func() interface{} {
+		return &blockBuffer{}
+	},
+}
+
+func releaseToValueBlockBufPool(pool *sync.Pool, b *blockBuffer) {
+	// Don't pool buffers larger than 128KB, in case we had some rare large
+	// values.
+	if len(b.b) > 128*1024 {
+		return
+	}
+	if invariants.Enabled {
+		// Set the bytes to a random value. Cap the number of bytes being
+		// randomized to prevent test timeouts.
+		length := cap(b.b)
+		if length > 1000 {
+			length = 1000
+		}
+		b.b = b.b[:length:length]
+		rand.Read(b.b)
+	}
+	pool.Put(b)
+}
+
+var valueBlockWriterPool = sync.Pool{
+	New: func() interface{} {
+		return &valueBlockWriter{}
+	},
+}
+
+func newValueBlockWriter(
+	blockSize int,
+	blockSizeThreshold int,
+	compression Compression,
+	checksumType ChecksumType,
+	// compressedSize should exclude the block trailer.
+	blockFinishedFunc func(compressedSize int),
+) *valueBlockWriter {
+	w := valueBlockWriterPool.Get().(*valueBlockWriter)
+	*w = valueBlockWriter{
+		blockSize:          blockSize,
+		blockSizeThreshold: blockSizeThreshold,
+		compression:        compression,
+		checksummer: checksummer{
+			checksumType: checksumType,
+		},
+		blockFinishedFunc: blockFinishedFunc,
+		buf:               uncompressedValueBlockBufPool.Get().(*blockBuffer),
+		compressedBuf:     compressedValueBlockBufPool.Get().(*blockBuffer),
+		blocks:            w.blocks[:0],
+	}
+	w.buf.b = w.buf.b[:0]
+	w.compressedBuf.b = w.compressedBuf.b[:0]
+	return w
+}
+
+func releaseValueBlockWriter(w *valueBlockWriter) {
+	for i := range w.blocks {
+		if w.blocks[i].compressed {
+			releaseToValueBlockBufPool(&compressedValueBlockBufPool, w.blocks[i].block)
+		} else {
+			releaseToValueBlockBufPool(&uncompressedValueBlockBufPool, w.blocks[i].block)
+		}
+		w.blocks[i].block = nil
+	}
+	if w.buf != nil {
+		releaseToValueBlockBufPool(&uncompressedValueBlockBufPool, w.buf)
+	}
+	if w.compressedBuf != nil {
+		releaseToValueBlockBufPool(&compressedValueBlockBufPool, w.compressedBuf)
+	}
+	*w = valueBlockWriter{
+		blocks: w.blocks[:0],
+	}
+	valueBlockWriterPool.Put(w)
+}
+
+func (w *valueBlockWriter) addValue(v []byte) (valueHandle, error) {
+	if invariants.Enabled && len(v) == 0 {
+		return valueHandle{}, errors.Errorf("cannot write empty value to value block")
+	}
+	w.numValues++
+	blockLen := len(w.buf.b)
+	valueLen := len(v)
+	if blockLen >= w.blockSize ||
+		(blockLen > w.blockSizeThreshold && blockLen+valueLen > w.blockSize) {
+		// Block is not currently empty and adding this value will become too big,
+		// so finish this block.
+		w.compressAndFlush()
+		blockLen = len(w.buf.b)
+		if invariants.Enabled && blockLen != 0 {
+			panic("blockLen of new block should be 0")
+		}
+	}
+	vh := valueHandle{
+		valueLen:      uint32(valueLen),
+		blockNum:      uint32(len(w.blocks)),
+		offsetInBlock: uint32(blockLen),
+	}
+	blockLen = int(vh.offsetInBlock + vh.valueLen)
+	if cap(w.buf.b) < blockLen {
+		size := 2 * cap(w.buf.b)
+		if size < 1024 {
+			size = 1024
+		}
+		for size < blockLen {
+			size *= 2
+		}
+		buf := make([]byte, blockLen, size)
+		_ = copy(buf, w.buf.b)
+		w.buf.b = buf
+	} else {
+		w.buf.b = w.buf.b[:blockLen]
+	}
+	buf := w.buf.b[vh.offsetInBlock:]
+	n := copy(buf, v)
+	if n != len(buf) {
+		panic("incorrect length computation")
+	}
+	return vh, nil
+}
+
+func (w *valueBlockWriter) compressAndFlush() {
+	// Compress the buffer, discarding the result if the improvement isn't at
+	// least 12.5%.
+	blockType := noCompressionBlockType
+	b := w.buf
+	if w.compression != NoCompression {
+		blockType, w.compressedBuf.b =
+			compressBlock(w.compression, w.buf.b, w.compressedBuf.b[:cap(w.compressedBuf.b)])
+		if len(w.compressedBuf.b) < len(w.buf.b)-len(w.buf.b)/8 {
+			b = w.compressedBuf
+		} else {
+			blockType = noCompressionBlockType
+		}
+	}
+	n := len(b.b)
+	if n+blockTrailerLen > cap(b.b) {
+		block := make([]byte, n+blockTrailerLen)
+		copy(block, b.b)
+		b.b = block
+	} else {
+		b.b = b.b[:n+blockTrailerLen]
+	}
+	b.b[n] = byte(blockType)
+	w.computeChecksum(b.b)
+	bh := BlockHandle{Offset: w.totalBlockBytes, Length: uint64(n)}
+	w.totalBlockBytes += uint64(len(b.b))
+	// blockFinishedFunc length excludes the block trailer.
+	w.blockFinishedFunc(n)
+	compressed := blockType != noCompressionBlockType
+	w.blocks = append(w.blocks, blockAndHandle{
+		block:      b,
+		handle:     bh,
+		compressed: compressed,
+	})
+	// Handed off a buffer to w.blocks, so need get a new one.
+	if compressed {
+		w.compressedBuf = compressedValueBlockBufPool.Get().(*blockBuffer)
+	} else {
+		w.buf = uncompressedValueBlockBufPool.Get().(*blockBuffer)
+	}
+	w.buf.b = w.buf.b[:0]
+}
+
+func (w *valueBlockWriter) computeChecksum(block []byte) {
+	n := len(block) - blockTrailerLen
+	checksum := w.checksummer.checksum(block[:n], block[n:n+1])
+	binary.LittleEndian.PutUint32(block[n+1:], checksum)
+}
+
+func (w *valueBlockWriter) finish(
+	writer io.Writer, fileOffset uint64,
+) (valueBlocksIndexHandle, valueBlocksAndIndexStats, error) {
+	if len(w.buf.b) > 0 {
+		w.compressAndFlush()
+	}
+	n := len(w.blocks)
+	if n == 0 {
+		return valueBlocksIndexHandle{}, valueBlocksAndIndexStats{}, nil
+	}
+	largestOffset := uint64(0)
+	largestLength := uint64(0)
+	for i := range w.blocks {
+		_, err := writer.Write(w.blocks[i].block.b)
+		if err != nil {
+			return valueBlocksIndexHandle{}, valueBlocksAndIndexStats{}, err
+		}
+		w.blocks[i].handle.Offset += fileOffset
+		largestOffset = w.blocks[i].handle.Offset
+		if largestLength < w.blocks[i].handle.Length {
+			largestLength = w.blocks[i].handle.Length
+		}
+	}
+	vbihOffset := fileOffset + w.totalBlockBytes
+
+	vbih := valueBlocksIndexHandle{
+		h: BlockHandle{
+			Offset: vbihOffset,
+		},
+		blockNumByteLength:    uint8(lenLittleEndian(uint64(n - 1))),
+		blockOffsetByteLength: uint8(lenLittleEndian(largestOffset)),
+		blockLengthByteLength: uint8(lenLittleEndian(largestLength)),
+	}
+	var err error
+	if vbih, err = w.writeValueBlocksIndex(writer, vbih); err != nil {
+		return valueBlocksIndexHandle{}, valueBlocksAndIndexStats{}, err
+	}
+	stats := valueBlocksAndIndexStats{
+		numValueBlocks:          uint64(n),
+		numValuesInValueBlocks:  w.numValues,
+		valueBlocksAndIndexSize: w.totalBlockBytes + vbih.h.Length + blockTrailerLen,
+	}
+	return vbih, stats, err
+}
+
+func (w *valueBlockWriter) writeValueBlocksIndex(
+	writer io.Writer, h valueBlocksIndexHandle,
+) (valueBlocksIndexHandle, error) {
+	blockLen :=
+		int(h.blockNumByteLength+h.blockOffsetByteLength+h.blockLengthByteLength) * len(w.blocks)
+	h.h.Length = uint64(blockLen)
+	blockLen += blockTrailerLen
+	var buf []byte
+	if cap(w.buf.b) < blockLen {
+		buf = make([]byte, blockLen)
+		w.buf.b = buf
+	} else {
+		buf = w.buf.b[:blockLen]
+	}
+	b := buf
+	for i := range w.blocks {
+		littleEndianPut(uint64(i), b, int(h.blockNumByteLength))
+		b = b[int(h.blockNumByteLength):]
+		littleEndianPut(w.blocks[i].handle.Offset, b, int(h.blockOffsetByteLength))
+		b = b[int(h.blockOffsetByteLength):]
+		littleEndianPut(w.blocks[i].handle.Length, b, int(h.blockLengthByteLength))
+		b = b[int(h.blockLengthByteLength):]
+	}
+	if len(b) != blockTrailerLen {
+		panic("incorrect length calculation")
+	}
+	b[0] = byte(noCompressionBlockType)
+	w.computeChecksum(buf)
+	if _, err := writer.Write(buf); err != nil {
+		return valueBlocksIndexHandle{}, err
+	}
+	return h, nil
+}
+
+// littleEndianPut writes v to b using little endian encoding, under the
+// assumption that v can be represented using n bytes.
+func littleEndianPut(v uint64, b []byte, n int) {
+	_ = b[n-1] // bounds check
+	for i := 0; i < n; i++ {
+		b[i] = byte(v)
+		v = v >> 8
+	}
+}
+
+// lenLittleEndian returns the minimum number of bytes needed to encode v
+// using little endian encoding.
+func lenLittleEndian(v uint64) int {
+	n := 0
+	for i := 0; i < 8; i++ {
+		n++
+		v = v >> 8
+		if v == 0 {
+			break
+		}
+	}
+	return n
+}
+
+func littleEndianGet(b []byte, n int) uint64 {
+	_ = b[n-1] // bounds check
+	v := uint64(b[0])
+	for i := 1; i < n; i++ {
+		v |= uint64(b[i]) << (8 * i)
+	}
+	return v
+}
+
+// UserKeyPrefixBound represents a [Lower,Upper) bound of user key prefixes.
+// If both are nil, there is no bound specified. Else, Compare(Lower,Upper)
+// must be < 0.
+type UserKeyPrefixBound struct {
+	// Lower is a lower bound user key prefix.
+	Lower []byte
+	// Upper is an upper bound user key prefix.
+	Upper []byte
+}
+
+// IsEmpty returns true iff the bound is empty.
+func (ukb *UserKeyPrefixBound) IsEmpty() bool {
+	return len(ukb.Lower) == 0 && len(ukb.Upper) == 0
+}
+
+type blockProviderWhenOpen interface {
+	readBlockForVBR(
+		ctx context.Context, h BlockHandle, stats *base.InternalIteratorStats,
+	) (cache.Handle, error)
+}
+
+type blockProviderWhenClosed struct {
+	rp ReaderProvider
+	r  *Reader
+}
+
+func (bpwc *blockProviderWhenClosed) open() error {
+	var err error
+	bpwc.r, err = bpwc.rp.GetReader()
+	return err
+}
+
+func (bpwc *blockProviderWhenClosed) close() {
+	bpwc.rp.Close()
+	bpwc.r = nil
+}
+
+func (bpwc blockProviderWhenClosed) readBlockForVBR(
+	ctx context.Context, h BlockHandle, stats *base.InternalIteratorStats,
+) (cache.Handle, error) {
+	ctx = objiotracing.WithBlockType(ctx, objiotracing.ValueBlock)
+	return bpwc.r.readBlock(ctx, h, nil, nil, stats)
+}
+
+// ReaderProvider supports the implementation of blockProviderWhenClosed.
+// GetReader and Close can be called multiple times in pairs.
+type ReaderProvider interface {
+	GetReader() (r *Reader, err error)
+	Close()
+}
+
+// TrivialReaderProvider implements ReaderProvider for a Reader that will
+// outlive the top-level iterator in the iterator tree.
+type TrivialReaderProvider struct {
+	*Reader
+}
+
+var _ ReaderProvider = TrivialReaderProvider{}
+
+// GetReader implements ReaderProvider.
+func (trp TrivialReaderProvider) GetReader() (*Reader, error) {
+	return trp.Reader, nil
+}
+
+// Close implements ReaderProvider.
+func (trp TrivialReaderProvider) Close() {}
+
+// valueBlockReader is used to retrieve values in value
+// blocks. It is used when the sstable was written with
+// Properties.ValueBlocksAreEnabled.
+type valueBlockReader struct {
+	ctx    context.Context
+	bpOpen blockProviderWhenOpen
+	rp     ReaderProvider
+	vbih   valueBlocksIndexHandle
+	stats  *base.InternalIteratorStats
+
+	// The value blocks index is lazily retrieved the first time the reader
+	// needs to read a value that resides in a value block.
+	vbiBlock []byte
+	vbiCache cache.Handle
+	// When sequentially iterating through all key-value pairs, the cost of
+	// repeatedly getting a block that is already in the cache and releasing the
+	// cache.Handle can be ~40% of the cpu overhead. So the reader remembers the
+	// last value block it retrieved, in case there is locality of access, and
+	// this value block can be used for the next value retrieval.
+	valueBlockNum uint32
+	valueBlock    []byte
+	valueBlockPtr unsafe.Pointer
+	valueCache    cache.Handle
+	lazyFetcher   base.LazyFetcher
+	closed        bool
+	bufToMangle   []byte
+}
+
+func (r *valueBlockReader) getLazyValueForPrefixAndValueHandle(handle []byte) base.LazyValue {
+	fetcher := &r.lazyFetcher
+	valLen, h := decodeLenFromValueHandle(handle[1:])
+	*fetcher = base.LazyFetcher{
+		Fetcher: r,
+		Attribute: base.AttributeAndLen{
+			ValueLen:       int32(valLen),
+			ShortAttribute: getShortAttribute(valuePrefix(handle[0])),
+		},
+	}
+	if r.stats != nil {
+		r.stats.SeparatedPointValue.Count++
+		r.stats.SeparatedPointValue.ValueBytes += uint64(valLen)
+	}
+	return base.LazyValue{
+		ValueOrHandle: h,
+		Fetcher:       fetcher,
+	}
+}
+
+func (r *valueBlockReader) close() {
+	r.bpOpen = nil
+	r.vbiBlock = nil
+	r.vbiCache.Release()
+	// Set the handle to empty since Release does not nil the Handle.value. If
+	// we were to reopen this valueBlockReader and retrieve the same
+	// Handle.value from the cache, we don't want to accidentally unref it when
+	// attempting to unref the old handle.
+	r.vbiCache = cache.Handle{}
+	r.valueBlock = nil
+	r.valueBlockPtr = nil
+	r.valueCache.Release()
+	// See comment above.
+	r.valueCache = cache.Handle{}
+	r.closed = true
+	// rp, vbih, stats remain valid, so that LazyFetcher.ValueFetcher can be
+	// implemented.
+}
+
+// Fetch implements base.ValueFetcher.
+func (r *valueBlockReader) Fetch(
+	handle []byte, valLen int32, buf []byte,
+) (val []byte, callerOwned bool, err error) {
+	if !r.closed {
+		val, err := r.getValueInternal(handle, valLen)
+		if invariants.Enabled {
+			val = r.doValueMangling(val)
+		}
+		return val, false, err
+	}
+
+	bp := blockProviderWhenClosed{rp: r.rp}
+	err = bp.open()
+	if err != nil {
+		return nil, false, err
+	}
+	defer bp.close()
+	defer r.close()
+	r.bpOpen = bp
+	var v []byte
+	v, err = r.getValueInternal(handle, valLen)
+	if err != nil {
+		return nil, false, err
+	}
+	buf = append(buf[:0], v...)
+	return buf, true, nil
+}
+
+// doValueMangling attempts to uncover violations of the contract listed in
+// the declaration comment of LazyValue. It is expensive, hence only called
+// when invariants.Enabled.
+func (r *valueBlockReader) doValueMangling(v []byte) []byte {
+	// Randomly set the bytes in the previous retrieved value to 0, since
+	// property P1 only requires the valueBlockReader to maintain the memory of
+	// one fetched value.
+	if rand.Intn(2) == 0 {
+		for i := range r.bufToMangle {
+			r.bufToMangle[i] = 0
+		}
+	}
+	// Store the current value in a new buffer for future mangling.
+	r.bufToMangle = append([]byte(nil), v...)
+	return r.bufToMangle
+}
+
+func (r *valueBlockReader) getValueInternal(handle []byte, valLen int32) (val []byte, err error) {
+	vh := decodeRemainingValueHandle(handle)
+	vh.valueLen = uint32(valLen)
+	if r.vbiBlock == nil {
+		ch, err := r.bpOpen.readBlockForVBR(r.ctx, r.vbih.h, r.stats)
+		if err != nil {
+			return nil, err
+		}
+		r.vbiCache = ch
+		r.vbiBlock = ch.Get()
+	}
+	if r.valueBlock == nil || r.valueBlockNum != vh.blockNum {
+		vbh, err := r.getBlockHandle(vh.blockNum)
+		if err != nil {
+			return nil, err
+		}
+		vbCacheHandle, err := r.bpOpen.readBlockForVBR(r.ctx, vbh, r.stats)
+		if err != nil {
+			return nil, err
+		}
+		r.valueBlockNum = vh.blockNum
+		r.valueCache.Release()
+		r.valueCache = vbCacheHandle
+		r.valueBlock = vbCacheHandle.Get()
+		r.valueBlockPtr = unsafe.Pointer(&r.valueBlock[0])
+	}
+	if r.stats != nil {
+		r.stats.SeparatedPointValue.ValueBytesFetched += uint64(valLen)
+	}
+	return r.valueBlock[vh.offsetInBlock : vh.offsetInBlock+vh.valueLen], nil
+}
+
+func (r *valueBlockReader) getBlockHandle(blockNum uint32) (BlockHandle, error) {
+	indexEntryLen :=
+		int(r.vbih.blockNumByteLength + r.vbih.blockOffsetByteLength + r.vbih.blockLengthByteLength)
+	offsetInIndex := indexEntryLen * int(blockNum)
+	if len(r.vbiBlock) < offsetInIndex+indexEntryLen {
+		return BlockHandle{}, errors.Errorf(
+			"cannot read at offset %d and length %d from block of length %d",
+			offsetInIndex, indexEntryLen, len(r.vbiBlock))
+	}
+	b := r.vbiBlock[offsetInIndex : offsetInIndex+indexEntryLen]
+	n := int(r.vbih.blockNumByteLength)
+	bn := littleEndianGet(b, n)
+	if uint32(bn) != blockNum {
+		return BlockHandle{},
+			errors.Errorf("expected block num %d but found %d", blockNum, bn)
+	}
+	b = b[n:]
+	n = int(r.vbih.blockOffsetByteLength)
+	blockOffset := littleEndianGet(b, n)
+	b = b[n:]
+	n = int(r.vbih.blockLengthByteLength)
+	blockLen := littleEndianGet(b, n)
+	return BlockHandle{Offset: blockOffset, Length: blockLen}, nil
+}

vendor/github.com/cockroachdb/pebble/sstable/writer.go

@@ -5,24 +5,25 @@
 package sstable
 
 import (
-	"bufio"
 	"bytes"
 	"encoding/binary"
 	"fmt"
-	"io"
 	"math"
 	"runtime"
+	"sort"
 	"sync"
 
 	"github.com/cespare/xxhash/v2"
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/bytealloc"
 	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/cockroachdb/pebble/internal/crc"
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/private"
 	"github.com/cockroachdb/pebble/internal/rangekey"
+	"github.com/cockroachdb/pebble/objstorage"
 )
 
 // encodedBHPEstimatedSize estimates the size of the encoded BlockHandleWithProperties.
@@ -109,27 +110,16 @@ func (m *WriterMetadata) updateSeqNum(seqNum uint64) {
 	}
 }
 
-type flusher interface {
-	Flush() error
-}
-
-type writeCloseSyncer interface {
-	io.WriteCloser
-	Sync() error
-}
-
 // Writer is a table writer.
 type Writer struct {
-	writer    io.Writer
-	bufWriter *bufio.Writer
-	syncer    writeCloseSyncer
-	meta      WriterMetadata
-	err       error
+	writable objstorage.Writable
+	meta     WriterMetadata
+	err      error
 	// cacheID and fileNum are used to remove blocks written to the sstable from
 	// the cache, providing a defense in depth against bugs which cause cache
 	// collisions.
 	cacheID uint64
-	fileNum base.FileNum
+	fileNum base.DiskFileNum
 	// The following fields are copied from Options.
 	blockSize               int
 	blockSizeThreshold      int
@@ -186,9 +176,9 @@ type Writer struct {
 	// indexBlockAlloc is used to bulk-allocate byte slices used to store index
 	// blocks in indexPartitions. These live until the index finishes.
 	indexBlockAlloc []byte
-	// indexSepAlloc is used to bulk-allocate index block seperator slices stored
+	// indexSepAlloc is used to bulk-allocate index block separator slices stored
 	// in indexPartitions. These live until the index finishes.
-	indexSepAlloc []byte
+	indexSepAlloc bytealloc.A
 
 	// To allow potentially overlapping (i.e. un-fragmented) range keys spans to
 	// be added to the Writer, a keyspan.Fragmenter is used to retain the keys
@@ -196,7 +186,8 @@ type Writer struct {
 	// keys must be added in order of their start user-key.
 	fragmenter        keyspan.Fragmenter
 	rangeKeyEncoder   rangekey.Encoder
-	rangeKeyCoalesced keyspan.Span
+	rangeKeysBySuffix keyspan.KeysBySuffix
+	rangeKeySpan      keyspan.Span
 	rkBuf             []byte
 	// dataBlockBuf consists of the state which is currently owned by and used by
 	// the Writer client goroutine. This state can be handed off to other goroutines.
@@ -206,6 +197,24 @@ type Writer struct {
 	blockBuf blockBuf
 
 	coordination coordinationState
+
+	// Information (other than the byte slice) about the last point key, to
+	// avoid extracting it again.
+	lastPointKeyInfo pointKeyInfo
+
+	// For value blocks.
+	shortAttributeExtractor   base.ShortAttributeExtractor
+	requiredInPlaceValueBound UserKeyPrefixBound
+	valueBlockWriter          *valueBlockWriter
+}
+
+type pointKeyInfo struct {
+	trailer uint64
+	// Only computed when w.valueBlockWriter is not nil.
+	userKeyLen int
+	// prefixLen uses w.split, if not nil. Only computed when w.valueBlockWriter
+	// is not nil.
+	prefixLen int
 }
 
 type coordinationState struct {
@@ -238,35 +247,37 @@ func (c *coordinationState) init(parallelismEnabled bool, writer *Writer) {
 
 // sizeEstimate is a general purpose helper for estimating two kinds of sizes:
 // A. The compressed sstable size, which is useful for deciding when to start
-//    a new sstable during flushes or compactions. In practice, we use this in
-//    estimating the data size (excluding the index).
+//
+//	a new sstable during flushes or compactions. In practice, we use this in
+//	estimating the data size (excluding the index).
+//
 // B. The size of index blocks to decide when to start a new index block.
 //
 // There are some terminology peculiarities which are due to the origin of
 // sizeEstimate for use case A with parallel compression enabled (for which
 // the code has not been merged). Specifically this relates to the terms
 // "written" and "compressed".
-// - The notion of "written" for case A is sufficiently defined by saying that
-//   the data block is compressed. Waiting for the actual data block write to
-//   happen can result in unnecessary estimation, when we already know how big
-//   it will be in compressed form. Additionally, with the forthcoming value
-//   blocks containing older MVCC values, these compressed block will be held
-//   in-memory until late in the sstable writing, and we do want to accurately
-//   account for them without waiting for the actual write.
-//   For case B, "written" means that the index entry has been fully
-//   generated, and has been added to the uncompressed block buffer for that
-//   index block. It does not include actually writing a potentially
-//   compressed index block.
-// - The notion of "compressed" is to differentiate between a "inflight" size
-//   and the actual size, and is handled via computing a compression ratio
-//   observed so far (defaults to 1).
-//   For case A, this is actual data block compression, so the "inflight" size
-//   is uncompressed blocks (that are no longer being written to) and the
-//   "compressed" size is after they have been compressed.
-//   For case B the inflight size is for a key-value pair in the index for
-//   which the value size (the encoded size of the BlockHandleWithProperties)
-//   is not accurately known, while the compressed size is the size of that
-//   entry when it has been added to the (in-progress) index ssblock.
+//   - The notion of "written" for case A is sufficiently defined by saying that
+//     the data block is compressed. Waiting for the actual data block write to
+//     happen can result in unnecessary estimation, when we already know how big
+//     it will be in compressed form. Additionally, with the forthcoming value
+//     blocks containing older MVCC values, these compressed block will be held
+//     in-memory until late in the sstable writing, and we do want to accurately
+//     account for them without waiting for the actual write.
+//     For case B, "written" means that the index entry has been fully
+//     generated, and has been added to the uncompressed block buffer for that
+//     index block. It does not include actually writing a potentially
+//     compressed index block.
+//   - The notion of "compressed" is to differentiate between a "inflight" size
+//     and the actual size, and is handled via computing a compression ratio
+//     observed so far (defaults to 1).
+//     For case A, this is actual data block compression, so the "inflight" size
+//     is uncompressed blocks (that are no longer being written to) and the
+//     "compressed" size is after they have been compressed.
+//     For case B the inflight size is for a key-value pair in the index for
+//     which the value size (the encoded size of the BlockHandleWithProperties)
+//     is not accurately known, while the compressed size is the size of that
+//     entry when it has been added to the (in-progress) index ssblock.
 //
 // Usage: To update state, one can optionally provide an inflight write value
 // using addInflight (used for case B). When something is "written" the state
@@ -727,40 +738,153 @@ func (w *Writer) Add(key InternalKey, value []byte) error {
 	return w.addPoint(key, value)
 }
 
-func (w *Writer) addPoint(key InternalKey, value []byte) error {
-	if !w.disableKeyOrderChecks && w.dataBlockBuf.dataBlock.nEntries >= 1 {
-		// curKey is guaranteed to be the last point key which was added to the Writer.
-		// Inlining base.DecodeInternalKey has a 2-3% improve in the BenchmarkWriter
-		// benchmark.
-		encodedKey := w.dataBlockBuf.dataBlock.curKey
-		n := len(encodedKey) - base.InternalTrailerLen
-		var trailer uint64
-		if n >= 0 {
-			trailer = binary.LittleEndian.Uint64(encodedKey[n:])
-			encodedKey = encodedKey[:n:n]
-		} else {
-			trailer = uint64(InternalKeyKindInvalid)
-			encodedKey = nil
+func (w *Writer) makeAddPointDecisionV2(key InternalKey) error {
+	prevTrailer := w.lastPointKeyInfo.trailer
+	w.lastPointKeyInfo.trailer = key.Trailer
+	if w.dataBlockBuf.dataBlock.nEntries == 0 {
+		return nil
+	}
+	if !w.disableKeyOrderChecks {
+		prevPointUserKey := w.dataBlockBuf.dataBlock.curKey
+		n := len(prevPointUserKey) - base.InternalTrailerLen
+		if n < 0 {
+			panic("corrupt key in blockWriter buffer")
 		}
-		largestPointKey := InternalKey{
-			UserKey: encodedKey,
-			Trailer: trailer,
-		}
-
-		if largestPointKey.UserKey != nil {
-			// TODO(peter): Manually inlined version of base.InternalCompare(). This is
-			// 3.5% faster on BenchmarkWriter on go1.13. Remove if go1.14 or future
-			// versions show this to not be a performance win.
-			x := w.compare(largestPointKey.UserKey, key.UserKey)
-			if x > 0 || (x == 0 && largestPointKey.Trailer <= key.Trailer) {
-				w.err = errors.Errorf("pebble: keys must be added in strictly increasing order: %s, %s",
-					largestPointKey.Pretty(w.formatKey), key.Pretty(w.formatKey))
-				return w.err
-			}
+		prevPointUserKey = prevPointUserKey[:n:n]
+		cmpUser := w.compare(prevPointUserKey, key.UserKey)
+		if cmpUser > 0 || (cmpUser == 0 && prevTrailer <= key.Trailer) {
+			return errors.Errorf(
+				"pebble: keys must be added in strictly increasing order: %s, %s",
+				InternalKey{UserKey: prevPointUserKey, Trailer: prevTrailer}.Pretty(w.formatKey),
+				key.Pretty(w.formatKey))
 		}
 	}
+	return nil
+}
 
-	if err := w.maybeFlush(key, value); err != nil {
+func (w *Writer) makeAddPointDecisionV3(
+	key InternalKey, valueLen int,
+) (setHasSamePrefix bool, writeToValueBlock bool, err error) {
+	prevPointKeyInfo := w.lastPointKeyInfo
+	w.lastPointKeyInfo.userKeyLen = len(key.UserKey)
+	w.lastPointKeyInfo.prefixLen = w.lastPointKeyInfo.userKeyLen
+	if w.split != nil {
+		w.lastPointKeyInfo.prefixLen = w.split(key.UserKey)
+	}
+	w.lastPointKeyInfo.trailer = key.Trailer
+	if w.dataBlockBuf.dataBlock.nEntries == 0 {
+		return false, false, nil
+	}
+	prevPointUserKey := w.dataBlockBuf.dataBlock.curKey
+	n := len(prevPointUserKey) - base.InternalTrailerLen
+	if n < 0 {
+		panic("corrupt key in blockWriter buffer")
+	}
+	prevPointUserKey = prevPointUserKey[:n:n]
+	prevPointKey := InternalKey{UserKey: prevPointUserKey, Trailer: prevPointKeyInfo.trailer}
+	considerWriteToValueBlock := base.TrailerKind(prevPointKeyInfo.trailer) == InternalKeyKindSet &&
+		base.TrailerKind(key.Trailer) == InternalKeyKindSet
+	if considerWriteToValueBlock && !w.requiredInPlaceValueBound.IsEmpty() {
+		keyPrefix := key.UserKey[:w.lastPointKeyInfo.prefixLen]
+		cmpUpper := w.compare(
+			w.requiredInPlaceValueBound.Upper, keyPrefix)
+		if cmpUpper <= 0 {
+			// Common case for CockroachDB. Make it empty since all future keys in
+			// this sstable will also have cmpUpper <= 0.
+			w.requiredInPlaceValueBound = UserKeyPrefixBound{}
+		} else if w.compare(keyPrefix, w.requiredInPlaceValueBound.Lower) >= 0 {
+			considerWriteToValueBlock = false
+		}
+	}
+	// cmpPrefix is initialized iff considerWriteToValueBlock.
+	var cmpPrefix int
+	var cmpUser int
+	if considerWriteToValueBlock {
+		// Compare the prefixes.
+		cmpPrefix = w.compare(prevPointUserKey[:prevPointKeyInfo.prefixLen],
+			key.UserKey[:w.lastPointKeyInfo.prefixLen])
+		cmpUser = cmpPrefix
+		if cmpPrefix == 0 {
+			// Need to compare suffixes to compute cmpUser.
+			cmpUser = w.compare(prevPointUserKey[prevPointKeyInfo.prefixLen:],
+				key.UserKey[w.lastPointKeyInfo.prefixLen:])
+		}
+	} else {
+		cmpUser = w.compare(prevPointUserKey, key.UserKey)
+	}
+	if !w.disableKeyOrderChecks &&
+		(cmpUser > 0 || (cmpUser == 0 && prevPointKeyInfo.trailer <= key.Trailer)) {
+		return false, false, errors.Errorf(
+			"pebble: keys must be added in strictly increasing order: %s, %s",
+			prevPointKey.Pretty(w.formatKey), key.Pretty(w.formatKey))
+	}
+	if !considerWriteToValueBlock {
+		return false, false, nil
+	}
+	// NB: it is possible that cmpUser == 0, i.e., these two SETs have identical
+	// user keys (because of an open snapshot). This should be the rare case.
+	setHasSamePrefix = cmpPrefix == 0
+	considerWriteToValueBlock = setHasSamePrefix
+	// Use of 0 here is somewhat arbitrary. Given the minimum 3 byte encoding of
+	// valueHandle, this should be > 3. But tiny values are common in test and
+	// unlikely in production, so we use 0 here for better test coverage.
+	const tinyValueThreshold = 0
+	if considerWriteToValueBlock && valueLen <= tinyValueThreshold {
+		considerWriteToValueBlock = false
+	}
+	return setHasSamePrefix, considerWriteToValueBlock, nil
+}
+
+func (w *Writer) addPoint(key InternalKey, value []byte) error {
+	var err error
+	var setHasSameKeyPrefix, writeToValueBlock, addPrefixToValueStoredWithKey bool
+	maxSharedKeyLen := len(key.UserKey)
+	if w.valueBlockWriter != nil {
+		// maxSharedKeyLen is limited to the prefix of the preceding key. If the
+		// preceding key was in a different block, then the blockWriter will
+		// ignore this maxSharedKeyLen.
+		maxSharedKeyLen = w.lastPointKeyInfo.prefixLen
+		setHasSameKeyPrefix, writeToValueBlock, err = w.makeAddPointDecisionV3(key, len(value))
+		addPrefixToValueStoredWithKey = base.TrailerKind(key.Trailer) == InternalKeyKindSet
+	} else {
+		err = w.makeAddPointDecisionV2(key)
+	}
+	if err != nil {
+		return err
+	}
+	var valueStoredWithKey []byte
+	var prefix valuePrefix
+	var valueStoredWithKeyLen int
+	if writeToValueBlock {
+		vh, err := w.valueBlockWriter.addValue(value)
+		if err != nil {
+			return err
+		}
+		n := encodeValueHandle(w.blockBuf.tmp[:], vh)
+		valueStoredWithKey = w.blockBuf.tmp[:n]
+		valueStoredWithKeyLen = len(valueStoredWithKey) + 1
+		var attribute base.ShortAttribute
+		if w.shortAttributeExtractor != nil {
+			// TODO(sumeer): for compactions, it is possible that the input sstable
+			// already has this value in the value section and so we have already
+			// extracted the ShortAttribute. Avoid extracting it again. This will
+			// require changing the Writer.Add interface.
+			if attribute, err = w.shortAttributeExtractor(
+				key.UserKey, w.lastPointKeyInfo.prefixLen, value); err != nil {
+				return err
+			}
+		}
+		prefix = makePrefixForValueHandle(setHasSameKeyPrefix, attribute)
+	} else {
+		valueStoredWithKey = value
+		valueStoredWithKeyLen = len(value)
+		if addPrefixToValueStoredWithKey {
+			valueStoredWithKeyLen++
+		}
+		prefix = makePrefixForInPlaceValue(setHasSameKeyPrefix)
+	}
+
+	if err := w.maybeFlush(key, valueStoredWithKeyLen); err != nil {
 		return err
 	}
 
@@ -771,14 +895,23 @@ func (w *Writer) addPoint(key InternalKey, value []byte) error {
 		}
 	}
 	for i := range w.blockPropCollectors {
-		if err := w.blockPropCollectors[i].Add(key, value); err != nil {
+		v := value
+		if addPrefixToValueStoredWithKey {
+			// Values for SET are not required to be in-place, and in the future may
+			// not even be read by the compaction, so pass nil values. Block
+			// property collectors in such Pebble DB's must not look at the value.
+			v = nil
+		}
+		if err := w.blockPropCollectors[i].Add(key, v); err != nil {
 			w.err = err
 			return err
 		}
 	}
 
 	w.maybeAddToFilter(key.UserKey)
-	w.dataBlockBuf.dataBlock.add(key, value)
+	w.dataBlockBuf.dataBlock.addWithOptionalValuePrefix(
+		key, valueStoredWithKey, maxSharedKeyLen, addPrefixToValueStoredWithKey, prefix,
+		setHasSameKeyPrefix)
 
 	w.meta.updateSeqNum(key.SeqNum())
 
@@ -797,8 +930,9 @@ func (w *Writer) addPoint(key InternalKey, value []byte) error {
 
 	w.props.NumEntries++
 	switch key.Kind() {
-	case InternalKeyKindDelete:
+	case InternalKeyKindDelete, InternalKeyKindSingleDelete:
 		w.props.NumDeletions++
+		w.props.RawPointTombstoneKeySize += uint64(key.Size())
 	case InternalKeyKindMerge:
 		w.props.NumMergeOperands++
 	}
@@ -903,10 +1037,15 @@ func (w *Writer) addTombstone(key InternalKey, value []byte) error {
 }
 
 // RangeKeySet sets a range between start (inclusive) and end (exclusive) with
-// the given suffix to the given value.
+// the given suffix to the given value. The resulting range key is given the
+// sequence number zero, with the expectation that the resulting sstable will be
+// ingested.
 //
 // Keys must be added to the table in increasing order of start key. Spans are
-// not required to be fragmented.
+// not required to be fragmented. The same suffix may not be set or unset twice
+// over the same keyspan, because it would result in inconsistent state. Both
+// the Set and Unset would share the zero sequence number, and a key cannot be
+// both simultaneously set and unset.
 func (w *Writer) RangeKeySet(start, end, suffix, value []byte) error {
 	return w.addRangeKeySpan(keyspan.Span{
 		Start: w.tempRangeKeyCopy(start),
@@ -922,10 +1061,15 @@ func (w *Writer) RangeKeySet(start, end, suffix, value []byte) error {
 }
 
 // RangeKeyUnset un-sets a range between start (inclusive) and end (exclusive)
-// with the given suffix.
+// with the given suffix. The resulting range key is given the
+// sequence number zero, with the expectation that the resulting sstable will be
+// ingested.
 //
 // Keys must be added to the table in increasing order of start key. Spans are
-// not required to be fragmented.
+// not required to be fragmented. The same suffix may not be set or unset twice
+// over the same keyspan, because it would result in inconsistent state. Both
+// the Set and Unset would share the zero sequence number, and a key cannot be
+// both simultaneously set and unset.
 func (w *Writer) RangeKeyUnset(start, end, suffix []byte) error {
 	return w.addRangeKeySpan(keyspan.Span{
 		Start: w.tempRangeKeyCopy(start),
@@ -970,6 +1114,11 @@ func (w *Writer) AddRangeKey(key InternalKey, value []byte) error {
 }
 
 func (w *Writer) addRangeKeySpan(span keyspan.Span) error {
+	if w.compare(span.Start, span.End) >= 0 {
+		return errors.Errorf(
+			"pebble: start key must be strictly less than end key",
+		)
+	}
 	if w.fragmenter.Start() != nil && w.compare(w.fragmenter.Start(), span.Start) > 0 {
 		return errors.Errorf("pebble: spans must be added in order: %s > %s",
 			w.formatKey(w.fragmenter.Start()), w.formatKey(span.Start))
@@ -979,20 +1128,23 @@ func (w *Writer) addRangeKeySpan(span keyspan.Span) error {
 	return w.err
 }
 
-func (w *Writer) coalesceSpans(span keyspan.Span) {
-	// This method is the emit function of the Fragmenter, so span.Keys is only
-	// owned by this span and it's safe to mutate.
-	w.rangeKeyCoalesced.Start = span.Start
-	w.rangeKeyCoalesced.End = span.End
-	err := rangekey.Coalesce(w.compare, span.Keys, &w.rangeKeyCoalesced.Keys)
-	if err != nil {
-		w.err = errors.Newf("sstable: could not coalesce span: %s", err)
-		return
-	}
+func (w *Writer) encodeRangeKeySpan(span keyspan.Span) {
+	// This method is the emit function of the Fragmenter.
+	//
+	// NB: The span should only contain range keys and be internally consistent
+	// (eg, no duplicate suffixes, no additional keys after a RANGEKEYDEL).
+	//
+	// We use w.rangeKeysBySuffix and w.rangeKeySpan to avoid allocations.
 
-	// NB: The span only contains range keys and is internally consistent (eg,
-	// no duplicate suffixes, no additional keys after a RANGEKEYDEL).
-	w.err = firstError(w.err, w.rangeKeyEncoder.Encode(&w.rangeKeyCoalesced))
+	// Sort the keys by suffix. Iteration doesn't *currently* depend on it, but
+	// we may want to in the future.
+	w.rangeKeysBySuffix.Cmp = w.compare
+	w.rangeKeysBySuffix.Keys = span.Keys
+	sort.Sort(&w.rangeKeysBySuffix)
+
+	w.rangeKeySpan = span
+	w.rangeKeySpan.Keys = w.rangeKeysBySuffix.Keys
+	w.err = firstError(w.err, w.rangeKeyEncoder.Encode(&w.rangeKeySpan))
 }
 
 func (w *Writer) addRangeKey(key InternalKey, value []byte) error {
@@ -1201,8 +1353,8 @@ func (w *Writer) flush(key InternalKey) error {
 	return err
 }
 
-func (w *Writer) maybeFlush(key InternalKey, value []byte) error {
-	if !w.dataBlockBuf.shouldFlush(key, len(value), w.blockSize, w.blockSizeThreshold) {
+func (w *Writer) maybeFlush(key InternalKey, valueLen int) error {
+	if !w.dataBlockBuf.shouldFlush(key, valueLen, w.blockSize, w.blockSizeThreshold) {
 		return nil
 	}
 
@@ -1274,11 +1426,11 @@ func (w *Writer) indexEntrySep(prevKey, key InternalKey, dataBlockBuf *dataBlock
 // they're used when the index block is finished.
 //
 // Invariant:
-// 1. addIndexEntry must not store references to the sep InternalKey, the tmp
-//    byte slice, bhp.Props. That is, these must be either deep copied or
-//    encoded.
-// 2. addIndexEntry must not hold references to the flushIndexBuf, and the writeTo
-//    indexBlockBufs.
+//  1. addIndexEntry must not store references to the sep InternalKey, the tmp
+//     byte slice, bhp.Props. That is, these must be either deep copied or
+//     encoded.
+//  2. addIndexEntry must not hold references to the flushIndexBuf, and the writeTo
+//     indexBlockBufs.
 func (w *Writer) addIndexEntry(
 	sep InternalKey,
 	bhp BlockHandleWithProperties,
@@ -1323,8 +1475,8 @@ func (w *Writer) addPrevDataBlockToIndexBlockProps() {
 // aren't being written asynchronously.
 //
 // Invariant:
-// 1. addIndexEntrySync must not store references to the prevKey, key InternalKey's,
-//    the tmp byte slice. That is, these must be either deep copied or encoded.
+//  1. addIndexEntrySync must not store references to the prevKey, key InternalKey's,
+//     the tmp byte slice. That is, these must be either deep copied or encoded.
 func (w *Writer) addIndexEntrySync(
 	prevKey, key InternalKey, bhp BlockHandleWithProperties, tmp []byte,
 ) error {
@@ -1388,25 +1540,22 @@ func shouldFlush(
 	return newSize > targetBlockSize
 }
 
-const keyAllocSize = 256 << 10
-
-func cloneKeyWithBuf(k InternalKey, buf []byte) ([]byte, InternalKey) {
+func cloneKeyWithBuf(k InternalKey, a bytealloc.A) (bytealloc.A, InternalKey) {
 	if len(k.UserKey) == 0 {
-		return buf, k
+		return a, k
 	}
-	if len(buf) < len(k.UserKey) {
-		buf = make([]byte, len(k.UserKey)+keyAllocSize)
-	}
-	n := copy(buf, k.UserKey)
-	return buf[n:], InternalKey{UserKey: buf[:n:n], Trailer: k.Trailer}
+	a, keyCopy := a.Copy(k.UserKey)
+	return a, InternalKey{UserKey: keyCopy, Trailer: k.Trailer}
 }
 
 // Invariants: The byte slice returned by finishIndexBlockProps is heap-allocated
-//  and has its own lifetime, independent of the Writer and the blockPropsEncoder,
+//
+//	and has its own lifetime, independent of the Writer and the blockPropsEncoder,
+//
 // and it is safe to:
-// 1. Reuse w.blockPropsEncoder without first encoding the byte slice returned.
-// 2. Store the byte slice in the Writer since it is a copy and not supported by
-//    an underlying buffer.
+//  1. Reuse w.blockPropsEncoder without first encoding the byte slice returned.
+//  2. Store the byte slice in the Writer since it is a copy and not supported by
+//     an underlying buffer.
 func (w *Writer) finishIndexBlockProps() ([]byte, error) {
 	w.blockPropsEncoder.resetProps()
 	for i := range w.blockPropCollectors {
@@ -1426,11 +1575,11 @@ func (w *Writer) finishIndexBlockProps() ([]byte, error) {
 // level index block. This is only used when two level indexes are enabled.
 //
 // Invariants:
-// 1. The props slice passed into finishedIndexBlock must not be a
-//    owned by any other struct, since it will be stored in the Writer.indexPartitions
-//    slice.
-// 2. None of the buffers owned by indexBuf will be shallow copied and stored elsewhere.
-//    That is, it must be safe to reuse indexBuf after finishIndexBlock has been called.
+//  1. The props slice passed into finishedIndexBlock must not be a
+//     owned by any other struct, since it will be stored in the Writer.indexPartitions
+//     slice.
+//  2. None of the buffers owned by indexBuf will be shallow copied and stored elsewhere.
+//     That is, it must be safe to reuse indexBuf after finishIndexBlock has been called.
 func (w *Writer) finishIndexBlock(indexBuf *indexBlockBuf, props []byte) error {
 	part := indexBlockAndBlockProperties{
 		nEntries: indexBuf.block.nEntries, properties: props,
@@ -1512,7 +1661,7 @@ func compressAndChecksum(b []byte, compression Compression, blockBuf *blockBuf)
 func (w *Writer) writeCompressedBlock(block []byte, blockTrailerBuf []byte) (BlockHandle, error) {
 	bh := BlockHandle{Offset: w.meta.Size, Length: uint64(len(block))}
 
-	if w.cacheID != 0 && w.fileNum != 0 {
+	if w.cacheID != 0 && w.fileNum.FileNum() != 0 {
 		// Remove the block being written from the cache. This provides defense in
 		// depth against bugs which cause cache collisions.
 		//
@@ -1522,20 +1671,38 @@ func (w *Writer) writeCompressedBlock(block []byte, blockTrailerBuf []byte) (Blo
 	}
 
 	// Write the bytes to the file.
-	n, err := w.writer.Write(block)
-	if err != nil {
+	if err := w.writable.Write(block); err != nil {
 		return BlockHandle{}, err
 	}
-	w.meta.Size += uint64(n)
-	n, err = w.writer.Write(blockTrailerBuf[:blockTrailerLen])
-	if err != nil {
+	w.meta.Size += uint64(len(block))
+	if err := w.writable.Write(blockTrailerBuf[:blockTrailerLen]); err != nil {
 		return BlockHandle{}, err
 	}
-	w.meta.Size += uint64(n)
+	w.meta.Size += blockTrailerLen
 
 	return bh, nil
 }
 
+// Write implements io.Writer. This is analogous to writeCompressedBlock for
+// blocks that already incorporate the trailer, and don't need the callee to
+// return a BlockHandle.
+func (w *Writer) Write(blockWithTrailer []byte) (n int, err error) {
+	offset := w.meta.Size
+	if w.cacheID != 0 && w.fileNum.FileNum() != 0 {
+		// Remove the block being written from the cache. This provides defense in
+		// depth against bugs which cause cache collisions.
+		//
+		// TODO(peter): Alternatively, we could add the uncompressed value to the
+		// cache.
+		w.cache.Delete(w.cacheID, w.fileNum, offset)
+	}
+	w.meta.Size += uint64(len(blockWithTrailer))
+	if err := w.writable.Write(blockWithTrailer); err != nil {
+		return 0, err
+	}
+	return len(blockWithTrailer), nil
+}
+
 func (w *Writer) writeBlock(
 	b []byte, compression Compression, blockBuf *blockBuf,
 ) (BlockHandle, error) {
@@ -1562,6 +1729,12 @@ func (w *Writer) assertFormatCompatibility() error {
 		)
 	}
 
+	if (w.props.NumValueBlocks > 0 || w.props.NumValuesInValueBlocks > 0 ||
+		w.props.ValueBlocksSize > 0) && w.tableFormat < TableFormatPebblev3 {
+		return errors.Newf(
+			"table format version %s is less than the minimum required version %s for value blocks",
+			w.tableFormat, TableFormatPebblev3)
+	}
 	return nil
 }
 
@@ -1569,21 +1742,28 @@ func (w *Writer) assertFormatCompatibility() error {
 // table was written to.
 func (w *Writer) Close() (err error) {
 	defer func() {
-		if w.syncer == nil {
-			return
+		if w.valueBlockWriter != nil {
+			releaseValueBlockWriter(w.valueBlockWriter)
+			// Defensive code in case Close gets called again. We don't want to put
+			// the same object to a sync.Pool.
+			w.valueBlockWriter = nil
 		}
-		err1 := w.syncer.Close()
-		if err == nil {
-			err = err1
+		if w.writable != nil {
+			w.writable.Abort()
+			w.writable = nil
+		}
+		// Record any error in the writer (so we can exit early if Close is called
+		// again).
+		if err != nil {
+			w.err = err
 		}
-		w.syncer = nil
 	}()
 
 	// finish must be called before we check for an error, because finish will
 	// block until every single task added to the writeQueue has been processed,
 	// and an error could be encountered while any of those tasks are processed.
-	if err = w.coordination.writeQueue.finish(); err != nil {
-		w.err = err
+	if err := w.coordination.writeQueue.finish(); err != nil {
+		return err
 	}
 
 	if w.err != nil {
@@ -1609,17 +1789,14 @@ func (w *Writer) Close() (err error) {
 	if w.dataBlockBuf.dataBlock.nEntries > 0 || w.indexBlock.block.nEntries == 0 {
 		bh, err := w.writeBlock(w.dataBlockBuf.dataBlock.finish(), w.compression, &w.dataBlockBuf.blockBuf)
 		if err != nil {
-			w.err = err
-			return w.err
+			return err
 		}
-		var bhp BlockHandleWithProperties
-		if bhp, err = w.maybeAddBlockPropertiesToBlockHandle(bh); err != nil {
-			w.err = err
+		bhp, err := w.maybeAddBlockPropertiesToBlockHandle(bh)
+		if err != nil {
 			return err
 		}
 		prevKey := base.DecodeInternalKey(w.dataBlockBuf.dataBlock.curKey)
-		if err = w.addIndexEntrySync(prevKey, InternalKey{}, bhp, w.dataBlockBuf.tmp[:]); err != nil {
-			w.err = err
+		if err := w.addIndexEntrySync(prevKey, InternalKey{}, bhp, w.dataBlockBuf.tmp[:]); err != nil {
 			return err
 		}
 	}
@@ -1631,13 +1808,11 @@ func (w *Writer) Close() (err error) {
 	if w.filter != nil {
 		b, err := w.filter.finish()
 		if err != nil {
-			w.err = err
-			return w.err
+			return err
 		}
 		bh, err := w.writeBlock(b, NoCompression, &w.blockBuf)
 		if err != nil {
-			w.err = err
-			return w.err
+			return err
 		}
 		n := encodeBlockHandle(w.blockBuf.tmp[:], bh)
 		metaindex.add(InternalKey{UserKey: []byte(w.filter.metaName())}, w.blockBuf.tmp[:n])
@@ -1651,8 +1826,7 @@ func (w *Writer) Close() (err error) {
 		// Write the two level index block.
 		indexBH, err = w.writeTwoLevelIndex()
 		if err != nil {
-			w.err = err
-			return w.err
+			return err
 		}
 	} else {
 		w.props.IndexType = binarySearchIndex
@@ -1665,8 +1839,7 @@ func (w *Writer) Close() (err error) {
 		// Write the single level index block.
 		indexBH, err = w.writeBlock(w.indexBlock.finish(), w.compression, &w.blockBuf)
 		if err != nil {
-			w.err = err
-			return w.err
+			return err
 		}
 	}
 
@@ -1691,8 +1864,7 @@ func (w *Writer) Close() (err error) {
 		}
 		rangeDelBH, err = w.writeBlock(w.rangeDelBlock.finish(), NoCompression, &w.blockBuf)
 		if err != nil {
-			w.err = err
-			return w.err
+			return err
 		}
 	}
 
@@ -1706,8 +1878,7 @@ func (w *Writer) Close() (err error) {
 		kind := key.Kind()
 		endKey, _, ok := rangekey.DecodeEndKey(kind, w.rangeKeyBlock.curValue)
 		if !ok {
-			w.err = errors.Newf("invalid end key: %s", w.rangeKeyBlock.curValue)
-			return w.err
+			return errors.Newf("invalid end key: %s", w.rangeKeyBlock.curValue)
 		}
 		k := base.MakeExclusiveSentinelKey(kind, endKey).Clone()
 		w.meta.SetLargestRangeKey(k)
@@ -1716,8 +1887,21 @@ func (w *Writer) Close() (err error) {
 		// enable compression on this block.
 		rangeKeyBH, err = w.writeBlock(w.rangeKeyBlock.finish(), NoCompression, &w.blockBuf)
 		if err != nil {
-			w.err = err
-			return w.err
+			return err
+		}
+	}
+
+	if w.valueBlockWriter != nil {
+		vbiHandle, vbStats, err := w.valueBlockWriter.finish(w, w.meta.Size)
+		if err != nil {
+			return err
+		}
+		w.props.NumValueBlocks = vbStats.numValueBlocks
+		w.props.NumValuesInValueBlocks = vbStats.numValuesInValueBlocks
+		w.props.ValueBlocksSize = vbStats.valueBlocksAndIndexSize
+		if vbStats.numValueBlocks > 0 {
+			n := encodeValueBlocksIndexHandle(w.blockBuf.tmp[:], vbiHandle)
+			metaindex.add(InternalKey{UserKey: []byte(metaValueIndexName)}, w.blockBuf.tmp[:n])
 		}
 	}
 
@@ -1734,7 +1918,6 @@ func (w *Writer) Close() (err error) {
 		userProps := make(map[string]string)
 		for i := range w.propCollectors {
 			if err := w.propCollectors[i].Finish(userProps); err != nil {
-				w.err = err
 				return err
 			}
 		}
@@ -1742,10 +1925,8 @@ func (w *Writer) Close() (err error) {
 			scratch := w.blockPropsEncoder.getScratchForProp()
 			// Place the shortID in the first byte.
 			scratch = append(scratch, byte(i))
-			buf, err :=
-				w.blockPropCollectors[i].FinishTable(scratch)
+			buf, err := w.blockPropCollectors[i].FinishTable(scratch)
 			if err != nil {
-				w.err = err
 				return err
 			}
 			var prop string
@@ -1768,11 +1949,10 @@ func (w *Writer) Close() (err error) {
 		// reduces table size without a significant impact on performance.
 		raw.restartInterval = propertiesBlockRestartInterval
 		w.props.CompressionOptions = rocksDBCompressionOptions
-		w.props.save(&raw)
+		w.props.save(w.tableFormat, &raw)
 		bh, err := w.writeBlock(raw.finish(), NoCompression, &w.blockBuf)
 		if err != nil {
-			w.err = err
-			return w.err
+			return err
 		}
 		n := encodeBlockHandle(w.blockBuf.tmp[:], bh)
 		metaindex.add(InternalKey{UserKey: []byte(metaPropertiesName)}, w.blockBuf.tmp[:n])
@@ -1798,8 +1978,7 @@ func (w *Writer) Close() (err error) {
 	// expect the meta-index block to not be compressed.
 	metaindexBH, err := w.writeBlock(metaindex.blockWriter.finish(), NoCompression, &w.blockBuf)
 	if err != nil {
-		w.err = err
-		return w.err
+		return err
 	}
 
 	// Write the table footer.
@@ -1809,34 +1988,25 @@ func (w *Writer) Close() (err error) {
 		metaindexBH: metaindexBH,
 		indexBH:     indexBH,
 	}
-	var n int
-	if n, err = w.writer.Write(footer.encode(w.blockBuf.tmp[:])); err != nil {
-		w.err = err
-		return w.err
+	encoded := footer.encode(w.blockBuf.tmp[:])
+	if err := w.writable.Write(footer.encode(w.blockBuf.tmp[:])); err != nil {
+		return err
 	}
-	w.meta.Size += uint64(n)
+	w.meta.Size += uint64(len(encoded))
 	w.meta.Properties = w.props
 
-	// Flush the buffer.
-	if w.bufWriter != nil {
-		if err := w.bufWriter.Flush(); err != nil {
-			w.err = err
-			return err
-		}
-	}
-
 	// Check that the features present in the table are compatible with the format
 	// configured for the table.
 	if err = w.assertFormatCompatibility(); err != nil {
-		w.err = err
-		return w.err
-	}
-
-	if err := w.syncer.Sync(); err != nil {
-		w.err = err
 		return err
 	}
 
+	if err := w.writable.Finish(); err != nil {
+		w.writable = nil
+		return err
+	}
+	w.writable = nil
+
 	w.dataBlockBuf.clear()
 	dataBlockBufPool.Put(w.dataBlockBuf)
 	w.dataBlockBuf = nil
@@ -1845,9 +2015,6 @@ func (w *Writer) Close() (err error) {
 	w.indexBlock = nil
 
 	// Make any future calls to Set or Close return an error.
-	if w.err != nil {
-		return w.err
-	}
 	w.err = errWriterClosed
 	return nil
 }
@@ -1863,7 +2030,7 @@ func (w *Writer) EstimatedSize() uint64 {
 // Metadata returns the metadata for the finished sstable. Only valid to call
 // after the sstable has been finished.
 func (w *Writer) Metadata() (*WriterMetadata, error) {
-	if w.syncer != nil {
+	if w.writable != nil {
 		return nil, errors.New("pebble: writer is not closed")
 	}
 	return &w.meta, nil
@@ -1905,23 +2072,12 @@ func (o *PreviousPointKeyOpt) writerApply(w *Writer) {
 	o.w = w
 }
 
-// internalTableOpt is a WriterOption that sets properties for sstables being
-// created by the db itself (i.e. through flushes and compactions), as opposed
-// to those meant for ingestion.
-type internalTableOpt struct{}
-
-func (i internalTableOpt) writerApply(w *Writer) {
-	// Set the external sst version to 0. This is what RocksDB expects for
-	// db-internal sstables; otherwise, it could apply a global sequence number.
-	w.props.ExternalFormatVersion = 0
-}
-
 // NewWriter returns a new table writer for the file. Closing the writer will
 // close the file.
-func NewWriter(f writeCloseSyncer, o WriterOptions, extraOpts ...WriterOption) *Writer {
+func NewWriter(writable objstorage.Writable, o WriterOptions, extraOpts ...WriterOption) *Writer {
 	o = o.ensureDefaults()
 	w := &Writer{
-		syncer: f,
+		writable: writable,
 		meta: WriterMetadata{
 			SmallestSeqNum: math.MaxUint64,
 		},
@@ -1954,6 +2110,14 @@ func NewWriter(f writeCloseSyncer, o WriterOptions, extraOpts ...WriterOption) *
 			Format: o.Comparer.FormatKey,
 		},
 	}
+	if w.tableFormat >= TableFormatPebblev3 {
+		w.shortAttributeExtractor = o.ShortAttributeExtractor
+		w.requiredInPlaceValueBound = o.RequiredInPlaceValueBound
+		w.valueBlockWriter = newValueBlockWriter(
+			w.blockSize, w.blockSizeThreshold, w.compression, w.checksumType, func(compressedSize int) {
+				w.coordination.sizeEstimate.dataBlockCompressed(compressedSize, 0)
+			})
+	}
 
 	w.dataBlockBuf = newDataBlockBuf(w.restartInterval, w.checksumType)
 
@@ -1963,13 +2127,13 @@ func NewWriter(f writeCloseSyncer, o WriterOptions, extraOpts ...WriterOption) *
 
 	w.coordination.init(o.Parallelism, w)
 
-	if f == nil {
-		w.err = errors.New("pebble: nil file")
+	if writable == nil {
+		w.err = errors.New("pebble: nil writable")
 		return w
 	}
 
 	// Note that WriterOptions are applied in two places; the ones with a
-	// preApply() method are applied here, and the rest are applied after
+	// preApply() method are applied here. The rest are applied down below after
 	// default properties are set.
 	type preApply interface{ preApply() }
 	for _, opt := range extraOpts {
@@ -2038,29 +2202,30 @@ func NewWriter(f writeCloseSyncer, o WriterOptions, extraOpts ...WriterOption) *
 
 	// Apply the remaining WriterOptions that do not have a preApply() method.
 	for _, opt := range extraOpts {
-		if _, ok := opt.(preApply); !ok {
-			opt.writerApply(w)
+		if _, ok := opt.(preApply); ok {
+			continue
 		}
+		opt.writerApply(w)
 	}
 
 	// Initialize the range key fragmenter and encoder.
-	w.fragmenter.Emit = w.coalesceSpans
+	w.fragmenter.Emit = w.encodeRangeKeySpan
 	w.rangeKeyEncoder.Emit = w.addRangeKey
-
-	// If f does not have a Flush method, do our own buffering.
-	if _, ok := f.(flusher); ok {
-		w.writer = f
-	} else {
-		w.bufWriter = bufio.NewWriter(f)
-		w.writer = w.bufWriter
-	}
 	return w
 }
 
+// internalGetProperties is a private, internal-use-only function that takes a
+// Writer and returns a pointer to its Properties, allowing direct mutation.
+// It's used by internal Pebble flushes and compactions to set internal
+// properties. It gets installed in private.
+func internalGetProperties(w *Writer) *Properties {
+	return &w.props
+}
+
 func init() {
 	private.SSTableWriterDisableKeyOrderChecks = func(i interface{}) {
 		w := i.(*Writer)
 		w.disableKeyOrderChecks = true
 	}
-	private.SSTableInternalTableOpt = internalTableOpt{}
+	private.SSTableInternalProperties = internalGetProperties
 }