Upgrade versions of vendored libs

vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go

@@ -18,22 +18,21 @@
 package arenaskl
 
 import (
-	"math"
 	"sync/atomic"
 	"unsafe"
 
 	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/constants"
+	"github.com/cockroachdb/pebble/internal/invariants"
 )
 
 // Arena is lock-free.
 type Arena struct {
-	n   uint64
+	n   atomic.Uint64
 	buf []byte
 }
 
-const (
-	align4 = 3
-)
+const nodeAlignment = 4
 
 var (
 	// ErrArenaFull indicates that the arena is full and cannot perform any more
@@ -44,20 +43,28 @@ var (
 // NewArena allocates a new arena using the specified buffer as the backing
 // store.
 func NewArena(buf []byte) *Arena {
-	// Don't store data at position 0 in order to reserve offset=0 as a kind
-	// of nil pointer.
-	return &Arena{
-		n:   1,
+	if len(buf) > constants.MaxUint32OrInt {
+		if invariants.Enabled {
+			panic(errors.AssertionFailedf("attempting to create arena of size %d", len(buf)))
+		}
+		buf = buf[:constants.MaxUint32OrInt]
+	}
+	a := &Arena{
 		buf: buf,
 	}
+	// We don't store data at position 0 in order to reserve offset=0 as a kind of
+	// nil pointer.
+	a.n.Store(1)
+	return a
 }
 
 // Size returns the number of bytes allocated by the arena.
 func (a *Arena) Size() uint32 {
-	s := atomic.LoadUint64(&a.n)
-	if s > math.MaxUint32 {
-		// Saturate at MaxUint32.
-		return math.MaxUint32
+	s := a.n.Load()
+	if s > constants.MaxUint32OrInt {
+		// The last failed allocation can push the size higher than len(a.buf).
+		// Saturate at the maximum representable offset.
+		return constants.MaxUint32OrInt
 	}
 	return uint32(s)
 }
@@ -67,24 +74,33 @@ func (a *Arena) Capacity() uint32 {
 	return uint32(len(a.buf))
 }
 
-func (a *Arena) alloc(size, align, overflow uint32) (uint32, uint32, error) {
+// alloc allocates a buffer of the given size and with the given alignment
+// (which must be a power of 2).
+//
+// If overflow is not 0, it also ensures that many bytes after the buffer are
+// inside the arena (this is used for structures that are larger than the
+// requested size but don't use those extra bytes).
+func (a *Arena) alloc(size, alignment, overflow uint32) (uint32, uint32, error) {
+	if invariants.Enabled && (alignment&(alignment-1)) != 0 {
+		panic(errors.AssertionFailedf("invalid alignment %d", alignment))
+	}
 	// Verify that the arena isn't already full.
-	origSize := atomic.LoadUint64(&a.n)
+	origSize := a.n.Load()
 	if int(origSize) > len(a.buf) {
 		return 0, 0, ErrArenaFull
 	}
 
 	// Pad the allocation with enough bytes to ensure the requested alignment.
-	padded := uint32(size) + align
+	padded := uint64(size) + uint64(alignment) - 1
 
-	newSize := atomic.AddUint64(&a.n, uint64(padded))
-	if int(newSize)+int(overflow) > len(a.buf) {
+	newSize := a.n.Add(padded)
+	if newSize+uint64(overflow) > uint64(len(a.buf)) {
 		return 0, 0, ErrArenaFull
 	}
 
 	// Return the aligned offset.
-	offset := (uint32(newSize) - padded + align) & ^align
-	return offset, padded, nil
+	offset := (uint32(newSize) - size) & ^(alignment - 1)
+	return offset, uint32(padded), nil
 }
 
 func (a *Arena) getBytes(offset uint32, size uint32) []byte {

vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go

@@ -18,7 +18,6 @@
 package arenaskl
 
 import (
-	"encoding/binary"
 	"sync"
 
 	"github.com/cockroachdb/pebble/internal/base"
@@ -240,17 +239,8 @@ func (it *Iterator) SetBounds(lower, upper []byte) {
 }
 
 func (it *Iterator) decodeKey() {
-	b := it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
-	// This is a manual inline of base.DecodeInternalKey, because the Go compiler
-	// seems to refuse to automatically inline it currently.
-	l := len(b) - 8
-	if l >= 0 {
-		it.key.Trailer = binary.LittleEndian.Uint64(b[l:])
-		it.key.UserKey = b[:l:l]
-	} else {
-		it.key.Trailer = uint64(base.InternalKeyKindInvalid)
-		it.key.UserKey = nil
-	}
+	it.key.UserKey = it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
+	it.key.Trailer = it.nd.keyTrailer
 }
 
 func (it *Iterator) seekForBaseSplice(key []byte) (prev, next *node, found bool) {

vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go

@@ -29,25 +29,27 @@ import (
 // is used here. If a key/value overflows a uint32, it should not be added to
 // the skiplist.
 func MaxNodeSize(keySize, valueSize uint32) uint64 {
-	return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + align4
+	const maxPadding = nodeAlignment - 1
+	return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + maxPadding
 }
 
 type links struct {
-	nextOffset uint32
-	prevOffset uint32
+	nextOffset atomic.Uint32
+	prevOffset atomic.Uint32
 }
 
 func (l *links) init(prevOffset, nextOffset uint32) {
-	l.nextOffset = nextOffset
-	l.prevOffset = prevOffset
+	l.nextOffset.Store(nextOffset)
+	l.prevOffset.Store(prevOffset)
 }
 
 type node struct {
 	// Immutable fields, so no need to lock to access key.
-	keyOffset uint32
-	keySize   uint32
-	valueSize uint32
-	allocSize uint32
+	keyOffset  uint32
+	keySize    uint32
+	keyTrailer uint64
+	valueSize  uint32
+	allocSize  uint32
 
 	// Most nodes do not need to use the full height of the tower, since the
 	// probability of each successive level decreases exponentially. Because
@@ -65,7 +67,7 @@ func newNode(
 	if height < 1 || height > maxHeight {
 		panic("height cannot be less than one or greater than the max height")
 	}
-	keySize := key.Size()
+	keySize := len(key.UserKey)
 	if int64(keySize) > math.MaxUint32 {
 		panic("key is too large")
 	}
@@ -81,8 +83,8 @@ func newNode(
 	if err != nil {
 		return
 	}
-
-	key.Encode(nd.getKeyBytes(arena))
+	nd.keyTrailer = key.Trailer
+	copy(nd.getKeyBytes(arena), key.UserKey)
 	copy(nd.getValue(arena), value)
 	return
 }
@@ -93,7 +95,7 @@ func newRawNode(arena *Arena, height uint32, keySize, valueSize uint32) (nd *nod
 	unusedSize := uint32((maxHeight - int(height)) * linksSize)
 	nodeSize := uint32(maxNodeSize) - unusedSize
 
-	nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, align4, unusedSize)
+	nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, nodeAlignment, unusedSize)
 	if err != nil {
 		return
 	}
@@ -115,17 +117,17 @@ func (n *node) getValue(arena *Arena) []byte {
 }
 
 func (n *node) nextOffset(h int) uint32 {
-	return atomic.LoadUint32(&n.tower[h].nextOffset)
+	return n.tower[h].nextOffset.Load()
 }
 
 func (n *node) prevOffset(h int) uint32 {
-	return atomic.LoadUint32(&n.tower[h].prevOffset)
+	return n.tower[h].prevOffset.Load()
 }
 
 func (n *node) casNextOffset(h int, old, val uint32) bool {
-	return atomic.CompareAndSwapUint32(&n.tower[h].nextOffset, old, val)
+	return n.tower[h].nextOffset.CompareAndSwap(old, val)
 }
 
 func (n *node) casPrevOffset(h int, old, val uint32) bool {
-	return atomic.CompareAndSwapUint32(&n.tower[h].prevOffset, old, val)
+	return n.tower[h].prevOffset.CompareAndSwap(old, val)
 }

vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go

@@ -44,7 +44,6 @@ Key differences:
 package arenaskl // import "github.com/cockroachdb/pebble/internal/arenaskl"
 
 import (
-	"encoding/binary"
 	"math"
 	"runtime"
 	"sync/atomic"
@@ -80,7 +79,7 @@ type Skiplist struct {
 	cmp    base.Compare
 	head   *node
 	tail   *node
-	height uint32 // Current height. 1 <= height <= maxHeight. CAS.
+	height atomic.Uint32 // Current height. 1 <= height <= maxHeight. CAS.
 
 	// If set to true by tests, then extra delays are added to make it easier to
 	// detect unusual race conditions.
@@ -140,22 +139,22 @@ func (s *Skiplist) Reset(arena *Arena, cmp base.Compare) {
 	headOffset := arena.getPointerOffset(unsafe.Pointer(head))
 	tailOffset := arena.getPointerOffset(unsafe.Pointer(tail))
 	for i := 0; i < maxHeight; i++ {
-		head.tower[i].nextOffset = tailOffset
-		tail.tower[i].prevOffset = headOffset
+		head.tower[i].nextOffset.Store(tailOffset)
+		tail.tower[i].prevOffset.Store(headOffset)
 	}
 
 	*s = Skiplist{
-		arena:  arena,
-		cmp:    cmp,
-		head:   head,
-		tail:   tail,
-		height: 1,
+		arena: arena,
+		cmp:   cmp,
+		head:  head,
+		tail:  tail,
 	}
+	s.height.Store(1)
 }
 
 // Height returns the height of the highest tower within any of the nodes that
 // have ever been allocated as part of this skiplist.
-func (s *Skiplist) Height() uint32 { return atomic.LoadUint32(&s.height) }
+func (s *Skiplist) Height() uint32 { return s.height.Load() }
 
 // Arena returns the arena backing this skiplist.
 func (s *Skiplist) Arena() *Arena { return s.arena }
@@ -327,7 +326,7 @@ func (s *Skiplist) newNode(
 	// Try to increase s.height via CAS.
 	listHeight := s.Height()
 	for height > listHeight {
-		if atomic.CompareAndSwapUint32(&s.height, listHeight, height) {
+		if s.height.CompareAndSwap(listHeight, height) {
 			// Successfully increased skiplist.height.
 			break
 		}
@@ -412,26 +411,19 @@ func (s *Skiplist) findSpliceForLevel(
 
 		offset, size := next.keyOffset, next.keySize
 		nextKey := s.arena.buf[offset : offset+size]
-		n := int32(size) - 8
-		cmp := s.cmp(key.UserKey, nextKey[:n])
+		cmp := s.cmp(key.UserKey, nextKey)
 		if cmp < 0 {
 			// We are done for this level, since prev.key < key < next.key.
 			break
 		}
 		if cmp == 0 {
 			// User-key equality.
-			var nextTrailer uint64
-			if n >= 0 {
-				nextTrailer = binary.LittleEndian.Uint64(nextKey[n:])
-			} else {
-				nextTrailer = uint64(base.InternalKeyKindInvalid)
-			}
-			if key.Trailer == nextTrailer {
+			if key.Trailer == next.keyTrailer {
 				// Internal key equality.
 				found = true
 				break
 			}
-			if key.Trailer > nextTrailer {
+			if key.Trailer > next.keyTrailer {
 				// We are done for this level, since prev.key < key < next.key.
 				break
 			}
@@ -446,8 +438,7 @@ func (s *Skiplist) findSpliceForLevel(
 
 func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
 	ndKey := s.arena.buf[nd.keyOffset : nd.keyOffset+nd.keySize]
-	n := int32(nd.keySize) - 8
-	cmp := s.cmp(ndKey[:n], key.UserKey)
+	cmp := s.cmp(ndKey, key.UserKey)
 	if cmp < 0 {
 		return true
 	}
@@ -455,25 +446,19 @@ func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
 		return false
 	}
 	// User-key equality.
-	var ndTrailer uint64
-	if n >= 0 {
-		ndTrailer = binary.LittleEndian.Uint64(ndKey[n:])
-	} else {
-		ndTrailer = uint64(base.InternalKeyKindInvalid)
-	}
-	if key.Trailer == ndTrailer {
+	if key.Trailer == nd.keyTrailer {
 		// Internal key equality.
 		return false
 	}
-	return key.Trailer < ndTrailer
+	return key.Trailer < nd.keyTrailer
 }
 
 func (s *Skiplist) getNext(nd *node, h int) *node {
-	offset := atomic.LoadUint32(&nd.tower[h].nextOffset)
+	offset := nd.tower[h].nextOffset.Load()
 	return (*node)(s.arena.getPointer(offset))
 }
 
 func (s *Skiplist) getPrev(nd *node, h int) *node {
-	offset := atomic.LoadUint32(&nd.tower[h].prevOffset)
+	offset := nd.tower[h].prevOffset.Load()
 	return (*node)(s.arena.getPointer(offset))
 }