Initial commit
This commit is contained in:
eleksir
411
vendor/github.com/cockroachdb/pebble/internal/rangekey/rangekey.go
generated
vendored
Normal file
411
vendor/github.com/cockroachdb/pebble/internal/rangekey/rangekey.go
generated
vendored
Normal file
@@ -0,0 +1,411 @@
|
||||
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
|
||||
// of this source code is governed by a BSD-style license that can be found in
|
||||
// the LICENSE file.
|
||||
|
||||
// Package rangekey provides facilities for encoding, decoding and merging range
|
||||
// keys.
|
||||
//
|
||||
// Range keys map a span of keyspan `[start, end)`, at an optional suffix, to a
|
||||
// value.
|
||||
//
|
||||
// # Encoding
|
||||
//
|
||||
// Unlike other Pebble keys, range keys encode several fields of information:
|
||||
// start key, end key, suffix and value. Internally within Pebble and its
|
||||
// sstables, all keys including range keys are represented as a key-value tuple.
|
||||
// Range keys map to internal key-value tuples by mapping the start key to the
|
||||
// key and encoding the remainder of the fields in the value.
|
||||
//
|
||||
// ## `RANGEKEYSET`
|
||||
//
|
||||
// A `RANGEKEYSET` represents one more range keys set over a single region of
|
||||
// user key space. Each represented range key must have a unique suffix. A
|
||||
// `RANGEKEYSET` encapsulates a start key, an end key and a set of SuffixValue
|
||||
// pairs.
|
||||
//
|
||||
// A `RANGEKEYSET` key's user key holds the start key. Its value is a varstring
|
||||
// end key, followed by a set of SuffixValue pairs. A `RANGEKEYSET` may have
|
||||
// multiple SuffixValue pairs if the keyspan was set at multiple unique suffix
|
||||
// values.
|
||||
//
|
||||
// ## `RANGEKEYUNSET`
|
||||
//
|
||||
// A `RANGEKEYUNSET` represents the removal of range keys at specific suffixes
|
||||
// over a single region of user key space. A `RANGEKEYUNSET` encapsulates a
|
||||
// start key, an end key and a set of suffixes.
|
||||
//
|
||||
// A `RANGEKEYUNSET` key's user key holds the start key. Its value is a
|
||||
// varstring end key, followed by a set of suffixes. A `RANGEKEYUNSET` may have
|
||||
// multiple suffixes if the keyspan was unset at multiple unique suffixes.
|
||||
//
|
||||
// ## `RANGEKEYDEL`
|
||||
//
|
||||
// A `RANGEKEYDEL` represents the removal of all range keys over a single region
|
||||
// of user key space, regardless of suffix. A `RANGEKEYDEL` encapsulates a
|
||||
// start key and an end key. The end key is stored in the value, without any
|
||||
// varstring length prefixing.
|
||||
package rangekey
|
||||
|
||||
// TODO(jackson): Document the encoding of RANGEKEYSET and RANGEKEYUNSET values
|
||||
// once we're confident they're stable.
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
|
||||
"github.com/cockroachdb/errors"
|
||||
"github.com/cockroachdb/pebble/internal/base"
|
||||
"github.com/cockroachdb/pebble/internal/keyspan"
|
||||
)
|
||||
|
||||
// Encode takes a Span containing only range keys. It invokes the provided
|
||||
// closure with the encoded internal keys that represent the Span's state. The
|
||||
// keys and values passed to emit are only valid until the closure returns.
|
||||
// If emit returns an error, Encode stops and returns the error.
|
||||
func Encode(s *keyspan.Span, emit func(k base.InternalKey, v []byte) error) error {
|
||||
enc := Encoder{Emit: emit}
|
||||
return enc.Encode(s)
|
||||
}
|
||||
|
||||
// An Encoder encodes range keys into their on-disk InternalKey format. An
|
||||
// Encoder holds internal buffers, reused between Emit calls.
|
||||
type Encoder struct {
|
||||
Emit func(base.InternalKey, []byte) error
|
||||
buf []byte
|
||||
unsets [][]byte
|
||||
sets []SuffixValue
|
||||
}
|
||||
|
||||
// Encode takes a Span containing only range keys. It invokes the Encoder's Emit
|
||||
// closure with the encoded internal keys that represent the Span's state. The
|
||||
// keys and values passed to emit are only valid until the closure returns. If
|
||||
// Emit returns an error, Encode stops and returns the error.
|
||||
//
|
||||
// The encoded key-value pair passed to Emit is only valid until the closure
|
||||
// completes.
|
||||
func (e *Encoder) Encode(s *keyspan.Span) error {
|
||||
if s.Empty() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// This for loop iterates through the span's keys, which are sorted by
|
||||
// sequence number descending, grouping them into sequence numbers. All keys
|
||||
// with identical sequence numbers are flushed together.
|
||||
var del bool
|
||||
var seqNum uint64
|
||||
for i := range s.Keys {
|
||||
if i == 0 || s.Keys[i].SeqNum() != seqNum {
|
||||
if i > 0 {
|
||||
// Flush all the existing internal keys that exist at seqNum.
|
||||
if err := e.flush(s, seqNum, del); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Reset sets, unsets, del.
|
||||
seqNum = s.Keys[i].SeqNum()
|
||||
del = false
|
||||
e.sets = e.sets[:0]
|
||||
e.unsets = e.unsets[:0]
|
||||
}
|
||||
|
||||
switch s.Keys[i].Kind() {
|
||||
case base.InternalKeyKindRangeKeySet:
|
||||
e.sets = append(e.sets, SuffixValue{
|
||||
Suffix: s.Keys[i].Suffix,
|
||||
Value: s.Keys[i].Value,
|
||||
})
|
||||
case base.InternalKeyKindRangeKeyUnset:
|
||||
e.unsets = append(e.unsets, s.Keys[i].Suffix)
|
||||
case base.InternalKeyKindRangeKeyDelete:
|
||||
del = true
|
||||
default:
|
||||
return base.CorruptionErrorf("pebble: %s key kind is not a range key", s.Keys[i].Kind())
|
||||
}
|
||||
}
|
||||
return e.flush(s, seqNum, del)
|
||||
}
|
||||
|
||||
// flush constructs internal keys for accumulated key state, and emits the
|
||||
// internal keys.
|
||||
func (e *Encoder) flush(s *keyspan.Span, seqNum uint64, del bool) error {
|
||||
if len(e.sets) > 0 {
|
||||
ik := base.MakeInternalKey(s.Start, seqNum, base.InternalKeyKindRangeKeySet)
|
||||
l := EncodedSetValueLen(s.End, e.sets)
|
||||
if l > cap(e.buf) {
|
||||
e.buf = make([]byte, l)
|
||||
}
|
||||
EncodeSetValue(e.buf[:l], s.End, e.sets)
|
||||
if err := e.Emit(ik, e.buf[:l]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(e.unsets) > 0 {
|
||||
ik := base.MakeInternalKey(s.Start, seqNum, base.InternalKeyKindRangeKeyUnset)
|
||||
l := EncodedUnsetValueLen(s.End, e.unsets)
|
||||
if l > cap(e.buf) {
|
||||
e.buf = make([]byte, l)
|
||||
}
|
||||
EncodeUnsetValue(e.buf[:l], s.End, e.unsets)
|
||||
if err := e.Emit(ik, e.buf[:l]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if del {
|
||||
ik := base.MakeInternalKey(s.Start, seqNum, base.InternalKeyKindRangeKeyDelete)
|
||||
// s.End is stored directly in the value for RangeKeyDeletes.
|
||||
if err := e.Emit(ik, s.End); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decode takes an internal key pair encoding range key(s) and returns a decoded
|
||||
// keyspan containing the keys. If keysDst is provided, keys will be appended to
|
||||
// keysDst.
|
||||
func Decode(ik base.InternalKey, v []byte, keysDst []keyspan.Key) (keyspan.Span, error) {
|
||||
var s keyspan.Span
|
||||
|
||||
// Hydrate the user key bounds.
|
||||
s.Start = ik.UserKey
|
||||
var ok bool
|
||||
s.End, v, ok = DecodeEndKey(ik.Kind(), v)
|
||||
if !ok {
|
||||
return keyspan.Span{}, base.CorruptionErrorf("pebble: unable to decode range key end from %s", ik.Kind())
|
||||
}
|
||||
s.Keys = keysDst
|
||||
|
||||
// Hydrate the contents of the range key(s).
|
||||
switch ik.Kind() {
|
||||
case base.InternalKeyKindRangeKeySet:
|
||||
for len(v) > 0 {
|
||||
var sv SuffixValue
|
||||
sv, v, ok = decodeSuffixValue(v)
|
||||
if !ok {
|
||||
return keyspan.Span{}, base.CorruptionErrorf("pebble: unable to decode range key suffix-value tuple")
|
||||
}
|
||||
s.Keys = append(s.Keys, keyspan.Key{
|
||||
Trailer: ik.Trailer,
|
||||
Suffix: sv.Suffix,
|
||||
Value: sv.Value,
|
||||
})
|
||||
}
|
||||
case base.InternalKeyKindRangeKeyUnset:
|
||||
for len(v) > 0 {
|
||||
var suffix []byte
|
||||
suffix, v, ok = decodeSuffix(v)
|
||||
if !ok {
|
||||
return keyspan.Span{}, base.CorruptionErrorf("pebble: unable to decode range key unset suffix")
|
||||
}
|
||||
s.Keys = append(s.Keys, keyspan.Key{
|
||||
Trailer: ik.Trailer,
|
||||
Suffix: suffix,
|
||||
})
|
||||
}
|
||||
case base.InternalKeyKindRangeKeyDelete:
|
||||
if len(v) > 0 {
|
||||
return keyspan.Span{}, base.CorruptionErrorf("pebble: RANGEKEYDELs must not contain additional data")
|
||||
}
|
||||
s.Keys = append(s.Keys, keyspan.Key{Trailer: ik.Trailer})
|
||||
default:
|
||||
return keyspan.Span{}, base.CorruptionErrorf("pebble: %s is not a range key", ik.Kind())
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// SuffixValue represents a tuple of a suffix and a corresponding value. A
|
||||
// physical RANGEKEYSET key may contain many logical RangeKeySets, each
|
||||
// represented with a separate SuffixValue tuple.
|
||||
type SuffixValue struct {
|
||||
Suffix []byte
|
||||
Value []byte
|
||||
}
|
||||
|
||||
// encodedSetSuffixValuesLen precomputes the length of the given slice of
|
||||
// SuffixValues, when encoded for a RangeKeySet. It may be used to construct a
|
||||
// buffer of the appropriate size before encoding.
|
||||
func encodedSetSuffixValuesLen(suffixValues []SuffixValue) int {
|
||||
var n int
|
||||
for i := 0; i < len(suffixValues); i++ {
|
||||
n += lenVarint(len(suffixValues[i].Suffix))
|
||||
n += len(suffixValues[i].Suffix)
|
||||
n += lenVarint(len(suffixValues[i].Value))
|
||||
n += len(suffixValues[i].Value)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// encodeSetSuffixValues encodes a slice of SuffixValues for a RangeKeySet into
|
||||
// dst. The length of dst must be greater than or equal to
|
||||
// encodedSetSuffixValuesLen. encodeSetSuffixValues returns the number of bytes
|
||||
// written, which should always equal the EncodedSetValueLen with the same
|
||||
// arguments.
|
||||
func encodeSetSuffixValues(dst []byte, suffixValues []SuffixValue) int {
|
||||
// Encode the list of (suffix, value-len) tuples.
|
||||
var n int
|
||||
for i := 0; i < len(suffixValues); i++ {
|
||||
// Encode the length of the suffix.
|
||||
n += binary.PutUvarint(dst[n:], uint64(len(suffixValues[i].Suffix)))
|
||||
|
||||
// Encode the suffix itself.
|
||||
n += copy(dst[n:], suffixValues[i].Suffix)
|
||||
|
||||
// Encode the value length.
|
||||
n += binary.PutUvarint(dst[n:], uint64(len(suffixValues[i].Value)))
|
||||
|
||||
// Encode the value itself.
|
||||
n += copy(dst[n:], suffixValues[i].Value)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// EncodedSetValueLen precomputes the length of a RangeKeySet's value when
|
||||
// encoded. It may be used to construct a buffer of the appropriate size before
|
||||
// encoding.
|
||||
func EncodedSetValueLen(endKey []byte, suffixValues []SuffixValue) int {
|
||||
n := lenVarint(len(endKey))
|
||||
n += len(endKey)
|
||||
n += encodedSetSuffixValuesLen(suffixValues)
|
||||
return n
|
||||
}
|
||||
|
||||
// EncodeSetValue encodes a RangeKeySet's value into dst. The length of dst must
|
||||
// be greater than or equal to EncodedSetValueLen. EncodeSetValue returns the
|
||||
// number of bytes written, which should always equal the EncodedSetValueLen
|
||||
// with the same arguments.
|
||||
func EncodeSetValue(dst []byte, endKey []byte, suffixValues []SuffixValue) int {
|
||||
// First encode the end key as a varstring.
|
||||
n := binary.PutUvarint(dst, uint64(len(endKey)))
|
||||
n += copy(dst[n:], endKey)
|
||||
n += encodeSetSuffixValues(dst[n:], suffixValues)
|
||||
return n
|
||||
}
|
||||
|
||||
// DecodeEndKey reads the end key from the beginning of a range key (RANGEKEYSET,
|
||||
// RANGEKEYUNSET or RANGEKEYDEL)'s physical encoded value. Both sets and unsets
|
||||
// encode the range key, plus additional data in the value.
|
||||
func DecodeEndKey(kind base.InternalKeyKind, data []byte) (endKey, value []byte, ok bool) {
|
||||
switch kind {
|
||||
case base.InternalKeyKindRangeKeyDelete:
|
||||
// No splitting is necessary for range key deletes. The value is the end
|
||||
// key, and there is no additional associated value.
|
||||
return data, nil, true
|
||||
case base.InternalKeyKindRangeKeySet, base.InternalKeyKindRangeKeyUnset:
|
||||
v, n := binary.Uvarint(data)
|
||||
if n <= 0 || uint64(n)+v >= uint64(len(data)) {
|
||||
return nil, nil, false
|
||||
}
|
||||
endKey, value = data[n:n+int(v)], data[n+int(v):]
|
||||
return endKey, value, true
|
||||
default:
|
||||
panic(errors.Newf("key kind %s is not a range key kind", kind))
|
||||
}
|
||||
}
|
||||
|
||||
// decodeSuffixValue decodes a single encoded SuffixValue from a RangeKeySet's
|
||||
// split value. The end key must have already been stripped from the
|
||||
// RangeKeySet's value (see DecodeEndKey).
|
||||
func decodeSuffixValue(data []byte) (sv SuffixValue, rest []byte, ok bool) {
|
||||
// Decode the suffix.
|
||||
sv.Suffix, data, ok = decodeVarstring(data)
|
||||
if !ok {
|
||||
return SuffixValue{}, nil, false
|
||||
}
|
||||
// Decode the value.
|
||||
sv.Value, data, ok = decodeVarstring(data)
|
||||
if !ok {
|
||||
return SuffixValue{}, nil, false
|
||||
}
|
||||
return sv, data, true
|
||||
}
|
||||
|
||||
// encodedUnsetSuffixesLen precomputes the length of the given slice of
|
||||
// suffixes, when encoded for a RangeKeyUnset. It may be used to construct a
|
||||
// buffer of the appropriate size before encoding.
|
||||
func encodedUnsetSuffixesLen(suffixes [][]byte) int {
|
||||
var n int
|
||||
for i := 0; i < len(suffixes); i++ {
|
||||
n += lenVarint(len(suffixes[i]))
|
||||
n += len(suffixes[i])
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// encodeUnsetSuffixes encodes a slice of suffixes for a RangeKeyUnset into dst.
|
||||
// The length of dst must be greater than or equal to EncodedUnsetSuffixesLen.
|
||||
// EncodeUnsetSuffixes returns the number of bytes written, which should always
|
||||
// equal the EncodedUnsetSuffixesLen with the same arguments.
|
||||
func encodeUnsetSuffixes(dst []byte, suffixes [][]byte) int {
|
||||
// Encode the list of (suffix, value-len) tuples.
|
||||
var n int
|
||||
for i := 0; i < len(suffixes); i++ {
|
||||
// Encode the length of the suffix.
|
||||
n += binary.PutUvarint(dst[n:], uint64(len(suffixes[i])))
|
||||
|
||||
// Encode the suffix itself.
|
||||
n += copy(dst[n:], suffixes[i])
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// EncodedUnsetValueLen precomputes the length of a RangeKeyUnset's value when
|
||||
// encoded. It may be used to construct a buffer of the appropriate size before
|
||||
// encoding.
|
||||
func EncodedUnsetValueLen(endKey []byte, suffixes [][]byte) int {
|
||||
n := lenVarint(len(endKey))
|
||||
n += len(endKey)
|
||||
n += encodedUnsetSuffixesLen(suffixes)
|
||||
return n
|
||||
}
|
||||
|
||||
// EncodeUnsetValue encodes a RangeKeyUnset's value into dst. The length of dst
|
||||
// must be greater than or equal to EncodedUnsetValueLen. EncodeUnsetValue
|
||||
// returns the number of bytes written, which should always equal the
|
||||
// EncodedUnsetValueLen with the same arguments.
|
||||
func EncodeUnsetValue(dst []byte, endKey []byte, suffixes [][]byte) int {
|
||||
// First encode the end key as a varstring.
|
||||
n := binary.PutUvarint(dst, uint64(len(endKey)))
|
||||
n += copy(dst[n:], endKey)
|
||||
n += encodeUnsetSuffixes(dst[n:], suffixes)
|
||||
return n
|
||||
}
|
||||
|
||||
// decodeSuffix decodes a single suffix from the beginning of data. If decoding
|
||||
// suffixes from a RangeKeyUnset's value, the end key must have already been
|
||||
// stripped from the RangeKeyUnset's value (see DecodeEndKey).
|
||||
func decodeSuffix(data []byte) (suffix, rest []byte, ok bool) {
|
||||
return decodeVarstring(data)
|
||||
}
|
||||
|
||||
func decodeVarstring(data []byte) (v, rest []byte, ok bool) {
|
||||
// Decode the length of the string.
|
||||
l, n := binary.Uvarint(data)
|
||||
if n <= 0 {
|
||||
return nil, nil, ok
|
||||
}
|
||||
|
||||
// Extract the string itself.
|
||||
return data[n : n+int(l)], data[n+int(l):], true
|
||||
}
|
||||
|
||||
// IsRangeKey returns true if the given key kind is one of the range key kinds.
|
||||
func IsRangeKey(kind base.InternalKeyKind) bool {
|
||||
switch kind {
|
||||
case base.InternalKeyKindRangeKeyDelete,
|
||||
base.InternalKeyKindRangeKeyUnset,
|
||||
base.InternalKeyKindRangeKeySet:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func lenVarint(v int) (n int) {
|
||||
x := uint32(v)
|
||||
n++
|
||||
for x >= 0x80 {
|
||||
x >>= 7
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
Reference in New Issue
Block a user