Upgrade versions of vendored libs

vendor/github.com/cockroachdb/pebble/sstable/block.go

@@ -8,10 +8,12 @@ import (
 	"encoding/binary"
 	"unsafe"
 
+	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
+	"github.com/cockroachdb/pebble/internal/manual"
 	"github.com/cockroachdb/pebble/internal/rangedel"
 	"github.com/cockroachdb/pebble/internal/rangekey"
 )
@@ -30,11 +32,33 @@ type blockWriter struct {
 	nEntries        int
 	nextRestart     int
 	buf             []byte
-	restarts        []uint32
-	curKey          []byte
-	curValue        []byte
-	prevKey         []byte
-	tmp             [4]byte
+	// For datablocks in TableFormatPebblev3, we steal the most significant bit
+	// in restarts for encoding setHasSameKeyPrefixSinceLastRestart. This leaves
+	// us with 31 bits, which is more than enough (no one needs > 2GB blocks).
+	// Typically, restarts occur every 16 keys, and by storing this bit with the
+	// restart, we can optimize for the case where a user wants to skip to the
+	// next prefix which happens to be in the same data block, but is > 16 keys
+	// away. We have seen production situations with 100+ versions per MVCC key
+	// (which share the same prefix). Additionally, for such writers, the prefix
+	// compression of the key, that shares the key with the preceding key, is
+	// limited to the prefix part of the preceding key -- this ensures that when
+	// doing NPrefix (see blockIter) we don't need to assemble the full key
+	// for each step since by limiting the length of the shared key we are
+	// ensuring that any of the keys with the same prefix can be used to
+	// assemble the full key when the prefix does change.
+	restarts []uint32
+	curKey   []byte
+	// curValue excludes the optional prefix provided to
+	// storeWithOptionalValuePrefix.
+	curValue []byte
+	prevKey  []byte
+	tmp      [4]byte
+	// We don't know the state of the sets that were at the end of the previous
+	// block, so this is initially 0. It may be true for the second and later
+	// restarts in a block. Not having inter-block information is fine since we
+	// will optimize by stepping through restarts only within the same block.
+	// Note that the first restart is the first key in the block.
+	setHasSameKeyPrefixSinceLastRestart bool
 }
 
 func (w *blockWriter) clear() {
@@ -47,17 +71,41 @@ func (w *blockWriter) clear() {
 	}
 }
 
-func (w *blockWriter) store(keySize int, value []byte) {
+// MaximumBlockSize is an extremely generous maximum block size of 256MiB. We
+// explicitly place this limit to reserve a few bits in the restart for
+// internal use.
+const MaximumBlockSize = 1 << 28
+const setHasSameKeyPrefixRestartMask uint32 = 1 << 31
+const restartMaskLittleEndianHighByteWithoutSetHasSamePrefix byte = 0b0111_1111
+const restartMaskLittleEndianHighByteOnlySetHasSamePrefix byte = 0b1000_0000
+
+// If !addValuePrefix, the valuePrefix is ignored.
+func (w *blockWriter) storeWithOptionalValuePrefix(
+	keySize int,
+	value []byte,
+	maxSharedKeyLen int,
+	addValuePrefix bool,
+	valuePrefix valuePrefix,
+	setHasSameKeyPrefix bool,
+) {
 	shared := 0
+	if !setHasSameKeyPrefix {
+		w.setHasSameKeyPrefixSinceLastRestart = false
+	}
 	if w.nEntries == w.nextRestart {
 		w.nextRestart = w.nEntries + w.restartInterval
-		w.restarts = append(w.restarts, uint32(len(w.buf)))
+		restart := uint32(len(w.buf))
+		if w.setHasSameKeyPrefixSinceLastRestart {
+			restart = restart | setHasSameKeyPrefixRestartMask
+		}
+		w.setHasSameKeyPrefixSinceLastRestart = true
+		w.restarts = append(w.restarts, restart)
 	} else {
 		// TODO(peter): Manually inlined version of base.SharedPrefixLen(). This
 		// is 3% faster on BenchmarkWriter on go1.16. Remove if future versions
 		// show this to not be a performance win. For now, functions that use of
 		// unsafe cannot be inlined.
-		n := len(w.curKey)
+		n := maxSharedKeyLen
 		if n > len(w.prevKey) {
 			n = len(w.prevKey)
 		}
@@ -72,7 +120,11 @@ func (w *blockWriter) store(keySize int, value []byte) {
 		}
 	}
 
-	needed := 3*binary.MaxVarintLen32 + len(w.curKey[shared:]) + len(value)
+	lenValuePlusOptionalPrefix := len(value)
+	if addValuePrefix {
+		lenValuePlusOptionalPrefix++
+	}
+	needed := 3*binary.MaxVarintLen32 + len(w.curKey[shared:]) + lenValuePlusOptionalPrefix
 	n := len(w.buf)
 	if cap(w.buf) < n+needed {
 		newCap := 2 * cap(w.buf)
@@ -114,7 +166,7 @@ func (w *blockWriter) store(keySize int, value []byte) {
 	}
 
 	{
-		x := uint32(len(value))
+		x := uint32(lenValuePlusOptionalPrefix)
 		for x >= 0x80 {
 			w.buf[n] = byte(x) | 0x80
 			x >>= 7
@@ -125,6 +177,10 @@ func (w *blockWriter) store(keySize int, value []byte) {
 	}
 
 	n += copy(w.buf[n:], w.curKey[shared:])
+	if addValuePrefix {
+		w.buf[n : n+1][0] = byte(valuePrefix)
+		n++
+	}
 	n += copy(w.buf[n:], value)
 	w.buf = w.buf[:n]
 
@@ -134,6 +190,19 @@ func (w *blockWriter) store(keySize int, value []byte) {
 }
 
 func (w *blockWriter) add(key InternalKey, value []byte) {
+	w.addWithOptionalValuePrefix(
+		key, value, len(key.UserKey), false, 0, false)
+}
+
+// Callers that always set addValuePrefix to false should use add() instead.
+func (w *blockWriter) addWithOptionalValuePrefix(
+	key InternalKey,
+	value []byte,
+	maxSharedKeyLen int,
+	addValuePrefix bool,
+	valuePrefix valuePrefix,
+	setHasSameKeyPrefix bool,
+) {
 	w.curKey, w.prevKey = w.prevKey, w.curKey
 
 	size := key.Size()
@@ -143,7 +212,8 @@ func (w *blockWriter) add(key InternalKey, value []byte) {
 	w.curKey = w.curKey[:size]
 	key.Encode(w.curKey)
 
-	w.store(size, value)
+	w.storeWithOptionalValuePrefix(
+		size, value, maxSharedKeyLen, addValuePrefix, valuePrefix, setHasSameKeyPrefix)
 }
 
 func (w *blockWriter) finish() []byte {
@@ -211,6 +281,36 @@ type blockEntry struct {
 // range keys since there is only a single range deletion and range key block
 // per sstable and the blockIter will not release the bytes for the block until
 // it is closed.
+//
+// Note on why blockIter knows about lazyValueHandling:
+//
+// blockIter's positioning functions (that return a LazyValue), are too
+// complex to inline even prior to lazyValueHandling. blockIter.Next and
+// blockIter.First were by far the cheapest and had costs 195 and 180
+// respectively, which exceeds the budget of 80. We initially tried to keep
+// the lazyValueHandling logic out of blockIter by wrapping it with a
+// lazyValueDataBlockIter. singleLevelIter and twoLevelIter would use this
+// wrapped iter. The functions in lazyValueDataBlockIter were simple, in that
+// they called the corresponding blockIter func and then decided whether the
+// value was in fact in-place (so return immediately) or needed further
+// handling. But these also turned out too costly for mid-stack inlining since
+// simple calls like the following have a high cost that is barely under the
+// budget of 80
+//
+//	k, v := i.data.SeekGE(key, flags)  // cost 74
+//	k, v := i.data.Next()              // cost 72
+//
+// We have 2 options for minimizing performance regressions:
+//   - Include the lazyValueHandling logic in the already non-inlineable
+//     blockIter functions: Since most of the time is spent in data block iters,
+//     it is acceptable to take the small hit of unnecessary branching (which
+//     hopefully branch prediction will predict correctly) for other kinds of
+//     blocks.
+//   - Duplicate the logic of singleLevelIterator and twoLevelIterator for the
+//     v3 sstable and only use the aforementioned lazyValueDataBlockIter for a
+//     v3 sstable. We would want to manage these copies via code generation.
+//
+// We have picked the first option here.
 type blockIter struct {
 	cmp Compare
 	// offset is the byte index that marks where the current key/value is
@@ -248,6 +348,9 @@ type blockIter struct {
 	// val contains the value the iterator is currently pointed at. If non-nil,
 	// this points to a slice of the block data.
 	val []byte
+	// lazyValue is val turned into a LazyValue, whenever a positioning method
+	// returns a non-nil key-value pair.
+	lazyValue base.LazyValue
 	// ikey contains the decoded InternalKey the iterator is currently pointed
 	// at. Note that the memory backing ikey.UserKey is either data stored
 	// directly in the block, fullKey, or cachedBuf. The key stability guarantee
@@ -273,7 +376,11 @@ type blockIter struct {
 	cacheHandle cache.Handle
 	// The first key in the block. This is used by the caller to set bounds
 	// for block iteration for already loaded blocks.
-	firstKey InternalKey
+	firstKey          InternalKey
+	lazyValueHandling struct {
+		vbr            *valueBlockReader
+		hasValuePrefix bool
+	}
 }
 
 // blockIter implements the base.InternalIterator interface.
@@ -413,6 +520,7 @@ func (i *blockIter) readEntry() {
 	}
 
 	unsharedKey := getBytes(ptr, int(unshared))
+	// TODO(sumeer): move this into the else block below.
 	i.fullKey = append(i.fullKey[:shared], unsharedKey...)
 	if shared == 0 {
 		// Provide stability for the key across positioning calls if the key
@@ -553,7 +661,7 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba
 		for index < upper {
 			h := int32(uint(index+upper) >> 1) // avoid overflow when computing h
 			// index ≤ h < upper
-			offset := int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*h:]))
+			offset := decodeRestart(i.data[i.restarts+4*h:])
 			// For a restart point, there are 0 bytes shared with the previous key.
 			// The varint encoding of 0 occupies 1 byte.
 			ptr := unsafe.Pointer(uintptr(i.ptr) + uintptr(offset+1))
@@ -621,18 +729,33 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba
 	// 0, then all keys in this block are larger than the key sought, and offset
 	// remains at zero.
 	if index > 0 {
-		i.offset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(index-1):]))
+		i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
 	}
 	i.readEntry()
 	i.decodeInternalKey(i.key)
 
 	// Iterate from that restart point to somewhere >= the key sought.
-	for ; i.valid(); i.Next() {
+	if !i.valid() {
+		return nil, base.LazyValue{}
+	}
+	if base.InternalCompare(i.cmp, i.ikey, ikey) >= 0 {
+		// Initialize i.lazyValue
+		if !i.lazyValueHandling.hasValuePrefix ||
+			base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+			i.lazyValue = base.MakeInPlaceValue(i.val)
+		} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+			i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+		} else {
+			i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+		}
+		return &i.ikey, i.lazyValue
+	}
+	for i.Next(); i.valid(); i.Next() {
 		if base.InternalCompare(i.cmp, i.ikey, ikey) >= 0 {
-			return &i.ikey, base.MakeInPlaceValue(i.val)
+			// i.Next() has already initialized i.lazyValue.
+			return &i.ikey, i.lazyValue
 		}
 	}
-
 	return nil, base.LazyValue{}
 }
 
@@ -666,7 +789,7 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
 		for index < upper {
 			h := int32(uint(index+upper) >> 1) // avoid overflow when computing h
 			// index ≤ h < upper
-			offset := int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*h:]))
+			offset := decodeRestart(i.data[i.restarts+4*h:])
 			// For a restart point, there are 0 bytes shared with the previous key.
 			// The varint encoding of 0 occupies 1 byte.
 			ptr := unsafe.Pointer(uintptr(i.ptr) + uintptr(offset+1))
@@ -733,9 +856,9 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
 	// index-1 will be the largest whose key is < the key sought.
 	targetOffset := i.restarts
 	if index > 0 {
-		i.offset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(index-1):]))
+		i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
 		if index < i.numRestarts {
-			targetOffset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(index):]))
+			targetOffset = decodeRestart(i.data[i.restarts+4*(index):])
 		}
 	} else if index == 0 {
 		// If index == 0 then all keys in this block are larger than the key
@@ -757,11 +880,12 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
 
 		if i.cmp(i.ikey.UserKey, ikey.UserKey) >= 0 {
 			// The current key is greater than or equal to our search key. Back up to
-			// the previous key which was less than our search key. Note that his for
+			// the previous key which was less than our search key. Note that this for
 			// loop will execute at least once with this if-block not being true, so
 			// the key we are backing up to is the last one this loop cached.
 			i.Prev()
-			return &i.ikey, base.MakeInPlaceValue(i.val)
+			// i.Prev() has already initialized i.lazyValue.
+			return &i.ikey, i.lazyValue
 		}
 
 		if i.nextOffset >= targetOffset {
@@ -779,7 +903,15 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba
 	if !i.valid() {
 		return nil, base.LazyValue{}
 	}
-	return &i.ikey, base.MakeInPlaceValue(i.val)
+	if !i.lazyValueHandling.hasValuePrefix ||
+		base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+		i.lazyValue = base.MakeInPlaceValue(i.val)
+	} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+		i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+	} else {
+		i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+	}
+	return &i.ikey, i.lazyValue
 }
 
 // First implements internalIterator.First, as documented in the pebble
@@ -792,13 +924,27 @@ func (i *blockIter) First() (*InternalKey, base.LazyValue) {
 	i.clearCache()
 	i.readEntry()
 	i.decodeInternalKey(i.key)
-	return &i.ikey, base.MakeInPlaceValue(i.val)
+	if !i.lazyValueHandling.hasValuePrefix ||
+		base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+		i.lazyValue = base.MakeInPlaceValue(i.val)
+	} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+		i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+	} else {
+		i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+	}
+	return &i.ikey, i.lazyValue
+}
+
+func decodeRestart(b []byte) int32 {
+	_ = b[3] // bounds check hint to compiler; see golang.org/issue/14808
+	return int32(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 |
+		uint32(b[3]&restartMaskLittleEndianHighByteWithoutSetHasSamePrefix)<<24)
 }
 
 // Last implements internalIterator.Last, as documented in the pebble package.
 func (i *blockIter) Last() (*InternalKey, base.LazyValue) {
 	// Seek forward from the last restart point.
-	i.offset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(i.numRestarts-1):]))
+	i.offset = decodeRestart(i.data[i.restarts+4*(i.numRestarts-1):])
 	if !i.valid() {
 		return nil, base.LazyValue{}
 	}
@@ -813,7 +959,15 @@ func (i *blockIter) Last() (*InternalKey, base.LazyValue) {
 	}
 
 	i.decodeInternalKey(i.key)
-	return &i.ikey, base.MakeInPlaceValue(i.val)
+	if !i.lazyValueHandling.hasValuePrefix ||
+		base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+		i.lazyValue = base.MakeInPlaceValue(i.val)
+	} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+		i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+	} else {
+		i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+	}
+	return &i.ikey, i.lazyValue
 }
 
 // Next implements internalIterator.Next, as documented in the pebble
@@ -850,7 +1004,307 @@ func (i *blockIter) Next() (*InternalKey, base.LazyValue) {
 		i.ikey.Trailer = uint64(InternalKeyKindInvalid)
 		i.ikey.UserKey = nil
 	}
-	return &i.ikey, base.MakeInPlaceValue(i.val)
+	if !i.lazyValueHandling.hasValuePrefix ||
+		base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+		i.lazyValue = base.MakeInPlaceValue(i.val)
+	} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+		i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+	} else {
+		i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+	}
+	return &i.ikey, i.lazyValue
+}
+
+// nextPrefix is used for implementing NPrefix.
+func (i *blockIter) nextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
+	if i.lazyValueHandling.hasValuePrefix {
+		return i.nextPrefixV3(succKey)
+	}
+	const nextsBeforeSeek = 3
+	k, v := i.Next()
+	for j := 1; k != nil && i.cmp(k.UserKey, succKey) < 0; j++ {
+		if j >= nextsBeforeSeek {
+			return i.SeekGE(succKey, base.SeekGEFlagsNone)
+		}
+		k, v = i.Next()
+	}
+	return k, v
+}
+
+func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue) {
+	// Doing nexts that involve a key comparison can be expensive (and the cost
+	// depends on the key length), so we use the same threshold of 3 that we use
+	// for TableFormatPebblev2 in blockIter.nextPrefix above. The next fast path
+	// that looks at setHasSamePrefix takes ~5ns per key, which is ~150x faster
+	// than doing a SeekGE within the block, so we do this 16 times
+	// (~5ns*16=80ns), and then switch to looking at restarts. Doing the binary
+	// search for the restart consumes > 100ns. If the number of versions is >
+	// 17, we will increment nextFastCount to 17, then do a binary search, and
+	// on average need to find a key between two restarts, so another 8 steps
+	// corresponding to nextFastCount, for a mean total of 17 + 8 = 25 such
+	// steps.
+	//
+	// TODO(sumeer): use the configured restartInterval for the sstable when it
+	// was written (which we don't currently store) instead of the default value
+	// of 16.
+	const nextCmpThresholdBeforeSeek = 3
+	const nextFastThresholdBeforeRestarts = 16
+	nextCmpCount := 0
+	nextFastCount := 0
+	usedRestarts := false
+	// INVARIANT: blockIter is valid.
+	if invariants.Enabled && !i.valid() {
+		panic(errors.AssertionFailedf("nextPrefixV3 called on invalid blockIter"))
+	}
+	prevKeyIsSet := i.ikey.Kind() == InternalKeyKindSet
+	for {
+		i.offset = i.nextOffset
+		if !i.valid() {
+			return nil, base.LazyValue{}
+		}
+		// Need to decode the length integers, so we can compute nextOffset.
+		ptr := unsafe.Pointer(uintptr(i.ptr) + uintptr(i.offset))
+		// This is an ugly performance hack. Reading entries from blocks is one of
+		// the inner-most routines and decoding the 3 varints per-entry takes
+		// significant time. Neither go1.11 or go1.12 will inline decodeVarint for
+		// us, so we do it manually. This provides a 10-15% performance improvement
+		// on blockIter benchmarks on both go1.11 and go1.12.
+		//
+		// TODO(peter): remove this hack if go:inline is ever supported.
+
+		// Decode the shared key length integer.
+		var shared uint32
+		if a := *((*uint8)(ptr)); a < 128 {
+			shared = uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 1)
+		} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
+			shared = uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 2)
+		} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
+			shared = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 3)
+		} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
+			shared = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 4)
+		} else {
+			d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
+			shared = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 5)
+		}
+		// Decode the unshared key length integer.
+		var unshared uint32
+		if a := *((*uint8)(ptr)); a < 128 {
+			unshared = uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 1)
+		} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
+			unshared = uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 2)
+		} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
+			unshared = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 3)
+		} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
+			unshared = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 4)
+		} else {
+			d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
+			unshared = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 5)
+		}
+		// Decode the value length integer.
+		var value uint32
+		if a := *((*uint8)(ptr)); a < 128 {
+			value = uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 1)
+		} else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 {
+			value = uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 2)
+		} else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 {
+			value = uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 3)
+		} else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 {
+			value = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 4)
+		} else {
+			d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4)))
+			value = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a)
+			ptr = unsafe.Pointer(uintptr(ptr) + 5)
+		}
+		// The starting position of the value.
+		valuePtr := unsafe.Pointer(uintptr(ptr) + uintptr(unshared))
+		i.nextOffset = int32(uintptr(valuePtr)-uintptr(i.ptr)) + int32(value)
+		if invariants.Enabled && unshared < 8 {
+			// This should not happen since only the key prefix is shared, so even
+			// if the prefix length is the same as the user key length, the unshared
+			// will include the trailer.
+			panic(errors.AssertionFailedf("unshared %d is too small", unshared))
+		}
+		// The trailer is written in little endian, so the key kind is the first
+		// byte in the trailer that is encoded in the slice [unshared-8:unshared].
+		keyKind := InternalKeyKind((*[manual.MaxArrayLen]byte)(ptr)[unshared-8])
+		prefixChanged := false
+		if keyKind == InternalKeyKindSet {
+			if invariants.Enabled && value == 0 {
+				panic(errors.AssertionFailedf("value is of length 0, but we expect a valuePrefix"))
+			}
+			valPrefix := *((*valuePrefix)(valuePtr))
+			if setHasSamePrefix(valPrefix) {
+				// Fast-path. No need to assemble i.fullKey, or update i.key. We know
+				// that subsequent keys will not have a shared length that is greater
+				// than the prefix of the current key, which is also the prefix of
+				// i.key. Since we are continuing to iterate, we don't need to
+				// initialize i.ikey and i.lazyValue (these are initialized before
+				// returning).
+				nextFastCount++
+				if nextFastCount > nextFastThresholdBeforeRestarts {
+					if usedRestarts {
+						// Exhausted iteration budget. This will never happen unless
+						// someone is using a restart interval > 16. It is just to guard
+						// against long restart intervals causing too much iteration.
+						break
+					}
+					// Haven't used restarts yet, so find the first restart at or beyond
+					// the current offset.
+					targetOffset := i.offset
+					var index int32
+					{
+						// NB: manually inlined sort.Sort is ~5% faster.
+						//
+						// f defined for a restart point is true iff the offset >=
+						// targetOffset.
+						// Define f(-1) == false and f(i.numRestarts) == true.
+						// Invariant: f(index-1) == false, f(upper) == true.
+						upper := i.numRestarts
+						for index < upper {
+							h := int32(uint(index+upper) >> 1) // avoid overflow when computing h
+							// index ≤ h < upper
+							offset := decodeRestart(i.data[i.restarts+4*h:])
+							if offset < targetOffset {
+								index = h + 1 // preserves f(index-1) == false
+							} else {
+								upper = h // preserves f(upper) == true
+							}
+						}
+						// index == upper, f(index-1) == false, and f(upper) (= f(index)) == true
+						// => answer is index.
+					}
+					usedRestarts = true
+					nextFastCount = 0
+					if index == i.numRestarts {
+						// Already past the last real restart, so iterate a bit more until
+						// we are done with the block.
+						continue
+					}
+					// Have some real restarts after index. NB: index is the first
+					// restart at or beyond the current offset.
+					startingIndex := index
+					for index != i.numRestarts &&
+						// The restart at index is 4 bytes written in little endian format
+						// starting at i.restart+4*index. The 0th byte is the least
+						// significant and the 3rd byte is the most significant. Since the
+						// most significant bit of the 3rd byte is what we use for
+						// encoding the set-has-same-prefix information, the indexing
+						// below has +3.
+						i.data[i.restarts+4*index+3]&restartMaskLittleEndianHighByteOnlySetHasSamePrefix != 0 {
+						// We still have the same prefix, so move to the next restart.
+						index++
+					}
+					// index is the first restart that did not have the same prefix.
+					if index != startingIndex {
+						// Managed to skip past at least one restart. Resume iteration
+						// from index-1. Since nextFastCount has been reset to 0, we
+						// should be able to iterate to the next prefix.
+						i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
+						i.readEntry()
+					}
+					// Else, unable to skip past any restart. Resume iteration. Since
+					// nextFastCount has been reset to 0, we should be able to iterate
+					// to the next prefix.
+					continue
+				}
+				continue
+			} else if prevKeyIsSet {
+				prefixChanged = true
+			}
+		} else {
+			prevKeyIsSet = false
+		}
+		// Slow-path cases:
+		// - (Likely) The prefix has changed.
+		// - (Unlikely) The prefix has not changed.
+		// We assemble the key etc. under the assumption that it is the likely
+		// case.
+		unsharedKey := getBytes(ptr, int(unshared))
+		// TODO(sumeer): move this into the else block below. This is a bit tricky
+		// since the current logic assumes we have always copied the latest key
+		// into fullKey, which is why when we get to the next key we can (a)
+		// access i.fullKey[:shared], (b) append only the unsharedKey to
+		// i.fullKey. For (a), we can access i.key[:shared] since that memory is
+		// valid (even if unshared). For (b), we will need to remember whether
+		// i.key refers to i.fullKey or not, and can append the unsharedKey only
+		// in the former case and for the latter case need to copy the shared part
+		// too. This same comment applies to the other place where we can do this
+		// optimization, in readEntry().
+		i.fullKey = append(i.fullKey[:shared], unsharedKey...)
+		i.val = getBytes(valuePtr, int(value))
+		if shared == 0 {
+			// Provide stability for the key across positioning calls if the key
+			// doesn't share a prefix with the previous key. This removes requiring the
+			// key to be copied if the caller knows the block has a restart interval of
+			// 1. An important example of this is range-del blocks.
+			i.key = unsharedKey
+		} else {
+			i.key = i.fullKey
+		}
+		// Manually inlined version of i.decodeInternalKey(i.key).
+		if n := len(i.key) - 8; n >= 0 {
+			i.ikey.Trailer = binary.LittleEndian.Uint64(i.key[n:])
+			i.ikey.UserKey = i.key[:n:n]
+			if i.globalSeqNum != 0 {
+				i.ikey.SetSeqNum(i.globalSeqNum)
+			}
+		} else {
+			i.ikey.Trailer = uint64(InternalKeyKindInvalid)
+			i.ikey.UserKey = nil
+		}
+		nextCmpCount++
+		if invariants.Enabled && prefixChanged && i.cmp(i.ikey.UserKey, succKey) < 0 {
+			panic(errors.AssertionFailedf("prefix should have changed but %x < %x",
+				i.ikey.UserKey, succKey))
+		}
+		if prefixChanged || i.cmp(i.ikey.UserKey, succKey) >= 0 {
+			// Prefix has changed.
+			if invariants.Enabled && !i.lazyValueHandling.hasValuePrefix {
+				panic(errors.AssertionFailedf("nextPrefixV3 being run for non-v3 sstable"))
+			}
+			if base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+				i.lazyValue = base.MakeInPlaceValue(i.val)
+			} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+				i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+			} else {
+				i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+			}
+			return &i.ikey, i.lazyValue
+		}
+		// Else prefix has not changed.
+
+		if nextCmpCount >= nextCmpThresholdBeforeSeek {
+			break
+		}
+	}
+	return i.SeekGE(succKey, base.SeekGEFlagsNone)
+}
+
+// NextPrefix implements (base.InternalIterator).NextPrefix
+func (i *blockIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) {
+	const nextsBeforeSeek = 3
+	k, v := i.Next()
+	for j := 1; k != nil && i.cmp(k.UserKey, succKey) < 0; j++ {
+		if j >= nextsBeforeSeek {
+			return i.SeekGE(succKey, base.SeekGEFlagsNone)
+		}
+		k, v = i.Next()
+	}
+	return k, v
 }
 
 // Prev implements internalIterator.Prev, as documented in the pebble
@@ -874,7 +1328,15 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
 			i.ikey.UserKey = nil
 		}
 		i.cached = i.cached[:n]
-		return &i.ikey, base.MakeInPlaceValue(i.val)
+		if !i.lazyValueHandling.hasValuePrefix ||
+			base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+			i.lazyValue = base.MakeInPlaceValue(i.val)
+		} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+			i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+		} else {
+			i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+		}
+		return &i.ikey, i.lazyValue
 	}
 
 	i.clearCache()
@@ -896,7 +1358,7 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
 		for index < upper {
 			h := int32(uint(index+upper) >> 1) // avoid overflow when computing h
 			// index ≤ h < upper
-			offset := int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*h:]))
+			offset := decodeRestart(i.data[i.restarts+4*h:])
 			if offset < targetOffset {
 				index = h + 1 // preserves f(i-1) == false
 			} else {
@@ -909,7 +1371,7 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
 
 	i.offset = 0
 	if index > 0 {
-		i.offset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(index-1):]))
+		i.offset = decodeRestart(i.data[i.restarts+4*(index-1):])
 	}
 
 	i.readEntry()
@@ -921,7 +1383,15 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) {
 	}
 
 	i.decodeInternalKey(i.key)
-	return &i.ikey, base.MakeInPlaceValue(i.val)
+	if !i.lazyValueHandling.hasValuePrefix ||
+		base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet {
+		i.lazyValue = base.MakeInPlaceValue(i.val)
+	} else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) {
+		i.lazyValue = base.MakeInPlaceValue(i.val[1:])
+	} else {
+		i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val)
+	}
+	return &i.ikey, i.lazyValue
 }
 
 // Key implements internalIterator.Key, as documented in the pebble package.
@@ -930,7 +1400,7 @@ func (i *blockIter) Key() *InternalKey {
 }
 
 func (i *blockIter) value() base.LazyValue {
-	return base.MakeInPlaceValue(i.val)
+	return i.lazyValue
 }
 
 // Error implements internalIterator.Error, as documented in the pebble
@@ -945,6 +1415,8 @@ func (i *blockIter) Close() error {
 	i.cacheHandle.Release()
 	i.cacheHandle = cache.Handle{}
 	i.val = nil
+	i.lazyValue = base.LazyValue{}
+	i.lazyValueHandling.vbr = nil
 	return nil
 }
 
@@ -966,7 +1438,7 @@ func (i *blockIter) valid() bool {
 // gathers all the fragments with identical bounds within a block and returns a
 // single keyspan.Span describing all the keys defined over the span.
 //
-// Memory lifetime
+// # Memory lifetime
 //
 // A Span returned by fragmentBlockIter is only guaranteed to be stable until
 // the next fragmentBlockIter iteration positioning method. A Span's Keys slice
@@ -1219,8 +1691,13 @@ func (i *fragmentBlockIter) Prev() *keyspan.Span {
 
 // SeekGE implements (keyspan.FragmentIterator).SeekGE.
 func (i *fragmentBlockIter) SeekGE(k []byte) *keyspan.Span {
-	i.dir = +1
-	return i.gatherForward(i.blockIter.SeekGE(k, base.SeekGEFlags(0)))
+	if s := i.SeekLT(k); s != nil && i.blockIter.cmp(k, s.End) < 0 {
+		return s
+	}
+	// TODO(jackson): If the above i.SeekLT(k) discovers a span but the span
+	// doesn't meet the k < s.End comparison, then there's no need for the
+	// SeekLT to gatherBackward.
+	return i.Next()
 }
 
 // SeekLT implements (keyspan.FragmentIterator).SeekLT.