aleesa-bot/aleesa-jabber-go
0
Fork 0
Code Activity

Initial commit

Browse Source
This commit is contained in:
eleksir
2023-09-17 12:18:09 +03:00
commit 843dd418b0
1514 changed files with 643531 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

201
vendor/github.com/cockroachdb/pebble/internal/arenaskl/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

93
vendor/github.com/cockroachdb/pebble/internal/arenaskl/README.md generated vendored Normal file
View File

@@ -0,0 +1,93 @@
# arenaskl
Fast, lock-free, arena-based Skiplist implementation in Go that supports iteration
in both directions.
## Advantages
Arenaskl offers several advantages over other skiplist implementations:
* High performance that linearly scales with the number of cores. This is
achieved by allocating from a fixed-size arena and by avoiding locks.
* Iterators that can be allocated on the stack and easily cloned by value.
* Simple-to-use and low overhead model for detecting and handling race conditions
with other threads.
* Support for iterating in reverse (i.e. previous links).
## Limitations
The advantages come at a cost that prevents arenaskl from being a general-purpose
skiplist implementation:
* The size of the arena sets a hard upper bound on the combined size of skiplist
nodes, keys, and values. This limit includes even the size of deleted nodes,
keys, and values.
* Deletion is not supported. Instead, higher-level code is expected to
add deletion tombstones and needs to process those tombstones
appropriately.
## Pedigree
This code is based on Andy Kimball's arenaskl code:
https://github.com/andy-kimball/arenaskl
The arenaskl code is based on the skiplist found in Badger, a Go-based
KV store:
https://github.com/dgraph-io/badger/tree/master/skl
The skiplist in Badger is itself based on a C++ skiplist built for
Facebook's RocksDB:
https://github.com/facebook/rocksdb/tree/master/memtable
## Benchmarks
The benchmarks consist of a mix of reads and writes executed in parallel. The
fraction of reads is indicated in the run name: "frac_X" indicates a run where
X percent of the operations are reads.
The results are much better than `skiplist` and `slist`.
```
name time/op
ReadWrite/frac_0-8 470ns ±11%
ReadWrite/frac_10-8 462ns ± 3%
ReadWrite/frac_20-8 436ns ± 2%
ReadWrite/frac_30-8 410ns ± 2%
ReadWrite/frac_40-8 385ns ± 2%
ReadWrite/frac_50-8 360ns ± 4%
ReadWrite/frac_60-8 386ns ± 1%
ReadWrite/frac_70-8 352ns ± 2%
ReadWrite/frac_80-8 306ns ± 3%
ReadWrite/frac_90-8 253ns ± 4%
ReadWrite/frac_100-8 28.1ns ± 2%
```
Note that the above numbers are for concurrent operations using 8x
parallelism. The same benchmarks without concurrency (use these
numbers when comparing vs batchskl):
```
name time/op
ReadWrite/frac_0 1.53µs ± 1%
ReadWrite/frac_10 1.46µs ± 2%
ReadWrite/frac_20 1.39µs ± 3%
ReadWrite/frac_30 1.28µs ± 3%
ReadWrite/frac_40 1.21µs ± 2%
ReadWrite/frac_50 1.11µs ± 3%
ReadWrite/frac_60 1.23µs ±17%
ReadWrite/frac_70 1.16µs ± 4%
ReadWrite/frac_80 959ns ± 3%
ReadWrite/frac_90 738ns ± 5%
ReadWrite/frac_100 81.9ns ± 2%
```
Forward and backward iteration are also fast:
```
name time/op
IterNext 3.97ns ± 5%
IterPrev 3.88ns ± 3%
```

125
vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go generated vendored Normal file
View File

@@ -0,0 +1,125 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package arenaskl
import (
"sync/atomic"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/constants"
"github.com/cockroachdb/pebble/internal/invariants"
)
// Arena is lock-free.
type Arena struct {
n atomic.Uint64
buf []byte
}
const nodeAlignment = 4
var (
// ErrArenaFull indicates that the arena is full and cannot perform any more
// allocations.
ErrArenaFull = errors.New("allocation failed because arena is full")
)
// NewArena allocates a new arena using the specified buffer as the backing
// store.
func NewArena(buf []byte) *Arena {
if len(buf) > constants.MaxUint32OrInt {
if invariants.Enabled {
panic(errors.AssertionFailedf("attempting to create arena of size %d", len(buf)))
}
buf = buf[:constants.MaxUint32OrInt]
}
a := &Arena{
buf: buf,
}
// We don't store data at position 0 in order to reserve offset=0 as a kind of
// nil pointer.
a.n.Store(1)
return a
}
// Size returns the number of bytes allocated by the arena.
func (a *Arena) Size() uint32 {
s := a.n.Load()
if s > constants.MaxUint32OrInt {
// The last failed allocation can push the size higher than len(a.buf).
// Saturate at the maximum representable offset.
return constants.MaxUint32OrInt
}
return uint32(s)
}
// Capacity returns the capacity of the arena.
func (a *Arena) Capacity() uint32 {
return uint32(len(a.buf))
}
// alloc allocates a buffer of the given size and with the given alignment
// (which must be a power of 2).
//
// If overflow is not 0, it also ensures that many bytes after the buffer are
// inside the arena (this is used for structures that are larger than the
// requested size but don't use those extra bytes).
func (a *Arena) alloc(size, alignment, overflow uint32) (uint32, uint32, error) {
if invariants.Enabled && (alignment&(alignment-1)) != 0 {
panic(errors.AssertionFailedf("invalid alignment %d", alignment))
}
// Verify that the arena isn't already full.
origSize := a.n.Load()
if int(origSize) > len(a.buf) {
return 0, 0, ErrArenaFull
}
// Pad the allocation with enough bytes to ensure the requested alignment.
padded := uint64(size) + uint64(alignment) - 1
newSize := a.n.Add(padded)
if newSize+uint64(overflow) > uint64(len(a.buf)) {
return 0, 0, ErrArenaFull
}
// Return the aligned offset.
offset := (uint32(newSize) - size) & ^(alignment - 1)
return offset, uint32(padded), nil
}
func (a *Arena) getBytes(offset uint32, size uint32) []byte {
if offset == 0 {
return nil
}
return a.buf[offset : offset+size : offset+size]
}
func (a *Arena) getPointer(offset uint32) unsafe.Pointer {
if offset == 0 {
return nil
}
return unsafe.Pointer(&a.buf[offset])
}
func (a *Arena) getPointerOffset(ptr unsafe.Pointer) uint32 {
if ptr == nil {
return 0
}
return uint32(uintptr(ptr) - uintptr(unsafe.Pointer(&a.buf[0])))
}

88
vendor/github.com/cockroachdb/pebble/internal/arenaskl/flush_iterator.go generated vendored Normal file
View File

@@ -0,0 +1,88 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package arenaskl
import "github.com/cockroachdb/pebble/internal/base"
// flushIterator is an iterator over the skiplist object. Use Skiplist.NewFlushIter
// to construct an iterator. The current state of the iterator can be cloned by
// simply value copying the struct.
type flushIterator struct {
Iterator
bytesIterated *uint64
}
// flushIterator implements the base.InternalIterator interface.
var _ base.InternalIterator = (*flushIterator)(nil)
func (it *flushIterator) String() string {
return "memtable"
}
func (it *flushIterator) SeekGE(
key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
panic("pebble: SeekGE unimplemented")
}
func (it *flushIterator) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
panic("pebble: SeekPrefixGE unimplemented")
}
func (it *flushIterator) SeekLT(
key []byte, flags base.SeekLTFlags,
) (*base.InternalKey, base.LazyValue) {
panic("pebble: SeekLT unimplemented")
}
// First seeks position at the first entry in list. Returns the key and value
// if the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
// that First only checks the upper bound. It is up to the caller to ensure
// that key is greater than or equal to the lower bound.
func (it *flushIterator) First() (*base.InternalKey, base.LazyValue) {
key, val := it.Iterator.First()
if key == nil {
return nil, base.LazyValue{}
}
*it.bytesIterated += uint64(it.nd.allocSize)
return key, val
}
// Next advances to the next position. Returns the key and value if the
// iterator is pointing at a valid entry, and (nil, nil) otherwise.
// Note: flushIterator.Next mirrors the implementation of Iterator.Next
// due to performance. Keep the two in sync.
func (it *flushIterator) Next() (*base.InternalKey, base.LazyValue) {
it.nd = it.list.getNext(it.nd, 0)
if it.nd == it.list.tail {
return nil, base.LazyValue{}
}
it.decodeKey()
*it.bytesIterated += uint64(it.nd.allocSize)
return &it.key, base.MakeInPlaceValue(it.value())
}
func (it *flushIterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
panic("pebble: NextPrefix unimplemented")
}
func (it *flushIterator) Prev() (*base.InternalKey, base.LazyValue) {
panic("pebble: Prev unimplemented")
}

271
vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go generated vendored Normal file
View File

@@ -0,0 +1,271 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package arenaskl
import (
"sync"
"github.com/cockroachdb/pebble/internal/base"
)
type splice struct {
prev *node
next *node
}
func (s *splice) init(prev, next *node) {
s.prev = prev
s.next = next
}
// Iterator is an iterator over the skiplist object. Use Skiplist.NewIter
// to construct an iterator. The current state of the iterator can be cloned by
// simply value copying the struct. All iterator methods are thread-safe.
type Iterator struct {
list *Skiplist
nd *node
key base.InternalKey
lower []byte
upper []byte
}
// Iterator implements the base.InternalIterator interface.
var _ base.InternalIterator = (*Iterator)(nil)
var iterPool = sync.Pool{
New: func() interface{} {
return &Iterator{}
},
}
// Close resets the iterator.
func (it *Iterator) Close() error {
it.list = nil
it.nd = nil
it.lower = nil
it.upper = nil
iterPool.Put(it)
return nil
}
func (it *Iterator) String() string {
return "memtable"
}
// Error returns any accumulated error.
func (it *Iterator) Error() error {
return nil
}
// SeekGE moves the iterator to the first entry whose key is greater than or
// equal to the given key. Returns the key and value if the iterator is
// pointing at a valid entry, and (nil, nil) otherwise. Note that SeekGE only
// checks the upper bound. It is up to the caller to ensure that key is greater
// than or equal to the lower bound.
func (it *Iterator) SeekGE(key []byte, flags base.SeekGEFlags) (*base.InternalKey, base.LazyValue) {
if flags.TrySeekUsingNext() {
if it.nd == it.list.tail {
// Iterator is done.
return nil, base.LazyValue{}
}
less := it.list.cmp(it.key.UserKey, key) < 0
// Arbitrary constant. By measuring the seek cost as a function of the
// number of elements in the skip list, and fitting to a model, we
// could adjust the number of nexts based on the current size of the
// skip list.
const numNexts = 5
for i := 0; less && i < numNexts; i++ {
k, _ := it.Next()
if k == nil {
// Iterator is done.
return nil, base.LazyValue{}
}
less = it.list.cmp(it.key.UserKey, key) < 0
}
if !less {
return &it.key, base.MakeInPlaceValue(it.value())
}
}
_, it.nd, _ = it.seekForBaseSplice(key)
if it.nd == it.list.tail {
return nil, base.LazyValue{}
}
it.decodeKey()
if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
it.nd = it.list.tail
return nil, base.LazyValue{}
}
return &it.key, base.MakeInPlaceValue(it.value())
}
// SeekPrefixGE moves the iterator to the first entry whose key is greater than
// or equal to the given key. This method is equivalent to SeekGE and is
// provided so that an arenaskl.Iterator implements the
// internal/base.InternalIterator interface.
func (it *Iterator) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
return it.SeekGE(key, flags)
}
// SeekLT moves the iterator to the last entry whose key is less than the given
// key. Returns the key and value if the iterator is pointing at a valid entry,
// and (nil, nil) otherwise. Note that SeekLT only checks the lower bound. It
// is up to the caller to ensure that key is less than the upper bound.
func (it *Iterator) SeekLT(key []byte, flags base.SeekLTFlags) (*base.InternalKey, base.LazyValue) {
// NB: the top-level Iterator has already adjusted key based on
// the upper-bound.
it.nd, _, _ = it.seekForBaseSplice(key)
if it.nd == it.list.head {
return nil, base.LazyValue{}
}
it.decodeKey()
if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
it.nd = it.list.head
return nil, base.LazyValue{}
}
return &it.key, base.MakeInPlaceValue(it.value())
}
// First seeks position at the first entry in list. Returns the key and value
// if the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
// that First only checks the upper bound. It is up to the caller to ensure
// that key is greater than or equal to the lower bound (e.g. via a call to SeekGE(lower)).
func (it *Iterator) First() (*base.InternalKey, base.LazyValue) {
it.nd = it.list.getNext(it.list.head, 0)
if it.nd == it.list.tail {
return nil, base.LazyValue{}
}
it.decodeKey()
if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
it.nd = it.list.tail
return nil, base.LazyValue{}
}
return &it.key, base.MakeInPlaceValue(it.value())
}
// Last seeks position at the last entry in list. Returns the key and value if
// the iterator is pointing at a valid entry, and (nil, nil) otherwise. Note
// that Last only checks the lower bound. It is up to the caller to ensure that
// key is less than the upper bound (e.g. via a call to SeekLT(upper)).
func (it *Iterator) Last() (*base.InternalKey, base.LazyValue) {
it.nd = it.list.getPrev(it.list.tail, 0)
if it.nd == it.list.head {
return nil, base.LazyValue{}
}
it.decodeKey()
if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
it.nd = it.list.head
return nil, base.LazyValue{}
}
return &it.key, base.MakeInPlaceValue(it.value())
}
// Next advances to the next position. Returns the key and value if the
// iterator is pointing at a valid entry, and (nil, nil) otherwise.
// Note: flushIterator.Next mirrors the implementation of Iterator.Next
// due to performance. Keep the two in sync.
func (it *Iterator) Next() (*base.InternalKey, base.LazyValue) {
it.nd = it.list.getNext(it.nd, 0)
if it.nd == it.list.tail {
return nil, base.LazyValue{}
}
it.decodeKey()
if it.upper != nil && it.list.cmp(it.upper, it.key.UserKey) <= 0 {
it.nd = it.list.tail
return nil, base.LazyValue{}
}
return &it.key, base.MakeInPlaceValue(it.value())
}
// NextPrefix advances to the next position with a new prefix. Returns the key
// and value if the iterator is pointing at a valid entry, and (nil, nil)
// otherwise.
func (it *Iterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
return it.SeekGE(succKey, base.SeekGEFlagsNone.EnableTrySeekUsingNext())
}
// Prev moves to the previous position. Returns the key and value if the
// iterator is pointing at a valid entry, and (nil, nil) otherwise.
func (it *Iterator) Prev() (*base.InternalKey, base.LazyValue) {
it.nd = it.list.getPrev(it.nd, 0)
if it.nd == it.list.head {
return nil, base.LazyValue{}
}
it.decodeKey()
if it.lower != nil && it.list.cmp(it.lower, it.key.UserKey) > 0 {
it.nd = it.list.head
return nil, base.LazyValue{}
}
return &it.key, base.MakeInPlaceValue(it.value())
}
// value returns the value at the current position.
func (it *Iterator) value() []byte {
return it.nd.getValue(it.list.arena)
}
// Head true iff the iterator is positioned at the sentinel head node.
func (it *Iterator) Head() bool {
return it.nd == it.list.head
}
// Tail true iff the iterator is positioned at the sentinel tail node.
func (it *Iterator) Tail() bool {
return it.nd == it.list.tail
}
// SetBounds sets the lower and upper bounds for the iterator. Note that the
// result of Next and Prev will be undefined until the iterator has been
// repositioned with SeekGE, SeekPrefixGE, SeekLT, First, or Last.
func (it *Iterator) SetBounds(lower, upper []byte) {
it.lower = lower
it.upper = upper
}
func (it *Iterator) decodeKey() {
it.key.UserKey = it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
it.key.Trailer = it.nd.keyTrailer
}
func (it *Iterator) seekForBaseSplice(key []byte) (prev, next *node, found bool) {
ikey := base.MakeSearchKey(key)
level := int(it.list.Height() - 1)
prev = it.list.head
for {
prev, next, found = it.list.findSpliceForLevel(ikey, level, prev)
if found {
if level != 0 {
// next is pointing at the target node, but we need to find previous on
// the bottom level.
prev = it.list.getPrev(next, 0)
}
break
}
if level == 0 {
break
}
level--
}
return
}

133
vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go generated vendored Normal file
View File

@@ -0,0 +1,133 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package arenaskl
import (
"math"
"sync/atomic"
"github.com/cockroachdb/pebble/internal/base"
)
// MaxNodeSize returns the maximum space needed for a node with the specified
// key and value sizes. This could overflow a uint32, which is why a uint64
// is used here. If a key/value overflows a uint32, it should not be added to
// the skiplist.
func MaxNodeSize(keySize, valueSize uint32) uint64 {
const maxPadding = nodeAlignment - 1
return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + maxPadding
}
type links struct {
nextOffset atomic.Uint32
prevOffset atomic.Uint32
}
func (l *links) init(prevOffset, nextOffset uint32) {
l.nextOffset.Store(nextOffset)
l.prevOffset.Store(prevOffset)
}
type node struct {
// Immutable fields, so no need to lock to access key.
keyOffset uint32
keySize uint32
keyTrailer uint64
valueSize uint32
allocSize uint32
// Most nodes do not need to use the full height of the tower, since the
// probability of each successive level decreases exponentially. Because
// these elements are never accessed, they do not need to be allocated.
// Therefore, when a node is allocated in the arena, its memory footprint
// is deliberately truncated to not include unneeded tower elements.
//
// All accesses to elements should use CAS operations, with no need to lock.
tower [maxHeight]links
}
func newNode(
arena *Arena, height uint32, key base.InternalKey, value []byte,
) (nd *node, err error) {
if height < 1 || height > maxHeight {
panic("height cannot be less than one or greater than the max height")
}
keySize := len(key.UserKey)
if int64(keySize) > math.MaxUint32 {
panic("key is too large")
}
valueSize := len(value)
if int64(len(value)) > math.MaxUint32 {
panic("value is too large")
}
if int64(len(value))+int64(keySize)+int64(maxNodeSize) > math.MaxUint32 {
panic("combined key and value size is too large")
}
nd, err = newRawNode(arena, height, uint32(keySize), uint32(valueSize))
if err != nil {
return
}
nd.keyTrailer = key.Trailer
copy(nd.getKeyBytes(arena), key.UserKey)
copy(nd.getValue(arena), value)
return
}
func newRawNode(arena *Arena, height uint32, keySize, valueSize uint32) (nd *node, err error) {
// Compute the amount of the tower that will never be used, since the height
// is less than maxHeight.
unusedSize := uint32((maxHeight - int(height)) * linksSize)
nodeSize := uint32(maxNodeSize) - unusedSize
nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, nodeAlignment, unusedSize)
if err != nil {
return
}
nd = (*node)(arena.getPointer(nodeOffset))
nd.keyOffset = nodeOffset + nodeSize
nd.keySize = keySize
nd.valueSize = valueSize
nd.allocSize = allocSize
return
}
func (n *node) getKeyBytes(arena *Arena) []byte {
return arena.getBytes(n.keyOffset, n.keySize)
}
func (n *node) getValue(arena *Arena) []byte {
return arena.getBytes(n.keyOffset+n.keySize, uint32(n.valueSize))
}
func (n *node) nextOffset(h int) uint32 {
return n.tower[h].nextOffset.Load()
}
func (n *node) prevOffset(h int) uint32 {
return n.tower[h].prevOffset.Load()
}
func (n *node) casNextOffset(h int, old, val uint32) bool {
return n.tower[h].nextOffset.CompareAndSwap(old, val)
}
func (n *node) casPrevOffset(h int, old, val uint32) bool {
return n.tower[h].prevOffset.CompareAndSwap(old, val)
}

464
vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go generated vendored Normal file
View File

@@ -0,0 +1,464 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
Adapted from RocksDB inline skiplist.
Key differences:
- No optimization for sequential inserts (no "prev").
- No custom comparator.
- Support overwrites. This requires care when we see the same key when inserting.
For RocksDB or LevelDB, overwrites are implemented as a newer sequence number in the key, so
there is no need for values. We don't intend to support versioning. In-place updates of values
would be more efficient.
- We discard all non-concurrent code.
- We do not support Splices. This simplifies the code a lot.
- No AllocateNode or other pointer arithmetic.
- We combine the findLessThan, findGreaterOrEqual, etc into one function.
*/
/*
Further adapted from Badger: https://github.com/dgraph-io/badger.
Key differences:
- Support for previous pointers - doubly linked lists. Note that it's up to higher
level code to deal with the intermediate state that occurs during insertion,
where node A is linked to node B, but node B is not yet linked back to node A.
- Iterator includes mutator functions.
*/
package arenaskl // import "github.com/cockroachdb/pebble/internal/arenaskl"
import (
"math"
"runtime"
"sync/atomic"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/fastrand"
)
const (
maxHeight = 20
maxNodeSize = int(unsafe.Sizeof(node{}))
linksSize = int(unsafe.Sizeof(links{}))
pValue = 1 / math.E
)
// ErrRecordExists indicates that an entry with the specified key already
// exists in the skiplist. Duplicate entries are not directly supported and
// instead must be handled by the user by appending a unique version suffix to
// keys.
var ErrRecordExists = errors.New("record with this key already exists")
// Skiplist is a fast, concurrent skiplist implementation that supports forward
// and backward iteration. See batchskl.Skiplist for a non-concurrent
// skiplist. Keys and values are immutable once added to the skiplist and
// deletion is not supported. Instead, higher-level code is expected to add new
// entries that shadow existing entries and perform deletion via tombstones. It
// is up to the user to process these shadow entries and tombstones
// appropriately during retrieval.
type Skiplist struct {
arena *Arena
cmp base.Compare
head *node
tail *node
height atomic.Uint32 // Current height. 1 <= height <= maxHeight. CAS.
// If set to true by tests, then extra delays are added to make it easier to
// detect unusual race conditions.
testing bool
}
// Inserter TODO(peter)
type Inserter struct {
spl [maxHeight]splice
height uint32
}
// Add TODO(peter)
func (ins *Inserter) Add(list *Skiplist, key base.InternalKey, value []byte) error {
return list.addInternal(key, value, ins)
}
var (
probabilities [maxHeight]uint32
)
func init() {
// Precompute the skiplist probabilities so that only a single random number
// needs to be generated and so that the optimal pvalue can be used (inverse
// of Euler's number).
p := float64(1.0)
for i := 0; i < maxHeight; i++ {
probabilities[i] = uint32(float64(math.MaxUint32) * p)
p *= pValue
}
}
// NewSkiplist constructs and initializes a new, empty skiplist. All nodes, keys,
// and values in the skiplist will be allocated from the given arena.
func NewSkiplist(arena *Arena, cmp base.Compare) *Skiplist {
skl := &Skiplist{}
skl.Reset(arena, cmp)
return skl
}
// Reset the skiplist to empty and re-initialize.
func (s *Skiplist) Reset(arena *Arena, cmp base.Compare) {
// Allocate head and tail nodes.
head, err := newRawNode(arena, maxHeight, 0, 0)
if err != nil {
panic("arenaSize is not large enough to hold the head node")
}
head.keyOffset = 0
tail, err := newRawNode(arena, maxHeight, 0, 0)
if err != nil {
panic("arenaSize is not large enough to hold the tail node")
}
tail.keyOffset = 0
// Link all head/tail levels together.
headOffset := arena.getPointerOffset(unsafe.Pointer(head))
tailOffset := arena.getPointerOffset(unsafe.Pointer(tail))
for i := 0; i < maxHeight; i++ {
head.tower[i].nextOffset.Store(tailOffset)
tail.tower[i].prevOffset.Store(headOffset)
}
*s = Skiplist{
arena: arena,
cmp: cmp,
head: head,
tail: tail,
}
s.height.Store(1)
}
// Height returns the height of the highest tower within any of the nodes that
// have ever been allocated as part of this skiplist.
func (s *Skiplist) Height() uint32 { return s.height.Load() }
// Arena returns the arena backing this skiplist.
func (s *Skiplist) Arena() *Arena { return s.arena }
// Size returns the number of bytes that have allocated from the arena.
func (s *Skiplist) Size() uint32 { return s.arena.Size() }
// Add adds a new key if it does not yet exist. If the key already exists, then
// Add returns ErrRecordExists. If there isn't enough room in the arena, then
// Add returns ErrArenaFull.
func (s *Skiplist) Add(key base.InternalKey, value []byte) error {
var ins Inserter
return s.addInternal(key, value, &ins)
}
func (s *Skiplist) addInternal(key base.InternalKey, value []byte, ins *Inserter) error {
if s.findSplice(key, ins) {
// Found a matching node, but handle case where it's been deleted.
return ErrRecordExists
}
if s.testing {
// Add delay to make it easier to test race between this thread
// and another thread that sees the intermediate state between
// finding the splice and using it.
runtime.Gosched()
}
nd, height, err := s.newNode(key, value)
if err != nil {
return err
}
ndOffset := s.arena.getPointerOffset(unsafe.Pointer(nd))
// We always insert from the base level and up. After you add a node in base
// level, we cannot create a node in the level above because it would have
// discovered the node in the base level.
var found bool
var invalidateSplice bool
for i := 0; i < int(height); i++ {
prev := ins.spl[i].prev
next := ins.spl[i].next
if prev == nil {
// New node increased the height of the skiplist, so assume that the
// new level has not yet been populated.
if next != nil {
panic("next is expected to be nil, since prev is nil")
}
prev = s.head
next = s.tail
}
// +----------------+ +------------+ +----------------+
// | prev | | nd | | next |
// | prevNextOffset |---->| | | |
// | |<----| prevOffset | | |
// | | | nextOffset |---->| |
// | | | |<----| nextPrevOffset |
// +----------------+ +------------+ +----------------+
//
// 1. Initialize prevOffset and nextOffset to point to prev and next.
// 2. CAS prevNextOffset to repoint from next to nd.
// 3. CAS nextPrevOffset to repoint from prev to nd.
for {
prevOffset := s.arena.getPointerOffset(unsafe.Pointer(prev))
nextOffset := s.arena.getPointerOffset(unsafe.Pointer(next))
nd.tower[i].init(prevOffset, nextOffset)
// Check whether next has an updated link to prev. If it does not,
// that can mean one of two things:
// 1. The thread that added the next node hasn't yet had a chance
// to add the prev link (but will shortly).
// 2. Another thread has added a new node between prev and next.
nextPrevOffset := next.prevOffset(i)
if nextPrevOffset != prevOffset {
// Determine whether #1 or #2 is true by checking whether prev
// is still pointing to next. As long as the atomic operations
// have at least acquire/release semantics (no need for
// sequential consistency), this works, as it is equivalent to
// the "publication safety" pattern.
prevNextOffset := prev.nextOffset(i)
if prevNextOffset == nextOffset {
// Ok, case #1 is true, so help the other thread along by
// updating the next node's prev link.
next.casPrevOffset(i, nextPrevOffset, prevOffset)
}
}
if prev.casNextOffset(i, nextOffset, ndOffset) {
// Managed to insert nd between prev and next, so update the next
// node's prev link and go to the next level.
if s.testing {
// Add delay to make it easier to test race between this thread
// and another thread that sees the intermediate state between
// setting next and setting prev.
runtime.Gosched()
}
next.casPrevOffset(i, prevOffset, ndOffset)
break
}
// CAS failed. We need to recompute prev and next. It is unlikely to
// be helpful to try to use a different level as we redo the search,
// because it is unlikely that lots of nodes are inserted between prev
// and next.
prev, next, found = s.findSpliceForLevel(key, i, prev)
if found {
if i != 0 {
panic("how can another thread have inserted a node at a non-base level?")
}
return ErrRecordExists
}
invalidateSplice = true
}
}
// If we had to recompute the splice for a level, invalidate the entire
// cached splice.
if invalidateSplice {
ins.height = 0
} else {
// The splice was valid. We inserted a node between spl[i].prev and
// spl[i].next. Optimistically update spl[i].prev for use in a subsequent
// call to add.
for i := uint32(0); i < height; i++ {
ins.spl[i].prev = nd
}
}
return nil
}
// NewIter returns a new Iterator object. The lower and upper bound parameters
// control the range of keys the iterator will return. Specifying for nil for
// lower or upper bound disables the check for that boundary. Note that lower
// bound is not checked on {SeekGE,First} and upper bound is not check on
// {SeekLT,Last}. The user is expected to perform that check. Note that it is
// safe for an iterator to be copied by value.
func (s *Skiplist) NewIter(lower, upper []byte) *Iterator {
it := iterPool.Get().(*Iterator)
*it = Iterator{list: s, nd: s.head, lower: lower, upper: upper}
return it
}
// NewFlushIter returns a new flushIterator, which is similar to an Iterator
// but also sets the current number of the bytes that have been iterated
// through.
func (s *Skiplist) NewFlushIter(bytesFlushed *uint64) base.InternalIterator {
return &flushIterator{
Iterator: Iterator{list: s, nd: s.head},
bytesIterated: bytesFlushed,
}
}
func (s *Skiplist) newNode(
key base.InternalKey, value []byte,
) (nd *node, height uint32, err error) {
height = s.randomHeight()
nd, err = newNode(s.arena, height, key, value)
if err != nil {
return
}
// Try to increase s.height via CAS.
listHeight := s.Height()
for height > listHeight {
if s.height.CompareAndSwap(listHeight, height) {
// Successfully increased skiplist.height.
break
}
listHeight = s.Height()
}
return
}
func (s *Skiplist) randomHeight() uint32 {
rnd := fastrand.Uint32()
h := uint32(1)
for h < maxHeight && rnd <= probabilities[h] {
h++
}
return h
}
func (s *Skiplist) findSplice(key base.InternalKey, ins *Inserter) (found bool) {
listHeight := s.Height()
var level int
prev := s.head
if ins.height < listHeight {
// Our cached height is less than the list height, which means there were
// inserts that increased the height of the list. Recompute the splice from
// scratch.
ins.height = listHeight
level = int(ins.height)
} else {
// Our cached height is equal to the list height.
for ; level < int(listHeight); level++ {
spl := &ins.spl[level]
if s.getNext(spl.prev, level) != spl.next {
// One or more nodes have been inserted between the splice at this
// level.
continue
}
if spl.prev != s.head && !s.keyIsAfterNode(spl.prev, key) {
// Key lies before splice.
level = int(listHeight)
break
}
if spl.next != s.tail && s.keyIsAfterNode(spl.next, key) {
// Key lies after splice.
level = int(listHeight)
break
}
// The splice brackets the key!
prev = spl.prev
break
}
}
for level = level - 1; level >= 0; level-- {
var next *node
prev, next, found = s.findSpliceForLevel(key, level, prev)
if next == nil {
next = s.tail
}
ins.spl[level].init(prev, next)
}
return
}
func (s *Skiplist) findSpliceForLevel(
key base.InternalKey, level int, start *node,
) (prev, next *node, found bool) {
prev = start
for {
// Assume prev.key < key.
next = s.getNext(prev, level)
if next == s.tail {
// Tail node, so done.
break
}
offset, size := next.keyOffset, next.keySize
nextKey := s.arena.buf[offset : offset+size]
cmp := s.cmp(key.UserKey, nextKey)
if cmp < 0 {
// We are done for this level, since prev.key < key < next.key.
break
}
if cmp == 0 {
// User-key equality.
if key.Trailer == next.keyTrailer {
// Internal key equality.
found = true
break
}
if key.Trailer > next.keyTrailer {
// We are done for this level, since prev.key < key < next.key.
break
}
}
// Keep moving right on this level.
prev = next
}
return
}
func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
ndKey := s.arena.buf[nd.keyOffset : nd.keyOffset+nd.keySize]
cmp := s.cmp(ndKey, key.UserKey)
if cmp < 0 {
return true
}
if cmp > 0 {
return false
}
// User-key equality.
if key.Trailer == nd.keyTrailer {
// Internal key equality.
return false
}
return key.Trailer < nd.keyTrailer
}
func (s *Skiplist) getNext(nd *node, h int) *node {
offset := nd.tower[h].nextOffset.Load()
return (*node)(s.arena.getPointer(offset))
}
func (s *Skiplist) getPrev(nd *node, h int) *node {
offset := nd.tower[h].prevOffset.Load()
return (*node)(s.arena.getPointer(offset))
}

60
vendor/github.com/cockroachdb/pebble/internal/base/cleaner.go generated vendored Normal file
View File

@@ -0,0 +1,60 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "github.com/cockroachdb/pebble/vfs"
// Cleaner cleans obsolete files.
type Cleaner interface {
Clean(fs vfs.FS, fileType FileType, path string) error
}
// NeedsFileContents is implemented by a cleaner that needs the contents of the
// files that it is being asked to clean.
type NeedsFileContents interface {
needsFileContents()
}
// DeleteCleaner deletes file.
type DeleteCleaner struct{}
// Clean removes file.
func (DeleteCleaner) Clean(fs vfs.FS, fileType FileType, path string) error {
return fs.Remove(path)
}
func (DeleteCleaner) String() string {
return "delete"
}
// ArchiveCleaner archives file instead delete.
type ArchiveCleaner struct{}
var _ NeedsFileContents = ArchiveCleaner{}
// Clean archives file.
func (ArchiveCleaner) Clean(fs vfs.FS, fileType FileType, path string) error {
switch fileType {
case FileTypeLog, FileTypeManifest, FileTypeTable:
destDir := fs.PathJoin(fs.PathDir(path), "archive")
if err := fs.MkdirAll(destDir, 0755); err != nil {
return err
}
destPath := fs.PathJoin(destDir, fs.PathBase(path))
return fs.Rename(path, destPath)
default:
return fs.Remove(path)
}
}
func (ArchiveCleaner) String() string {
return "archive"
}
func (ArchiveCleaner) needsFileContents() {
}

260
vendor/github.com/cockroachdb/pebble/internal/base/comparer.go generated vendored Normal file
View File

@@ -0,0 +1,260 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import (
"bytes"
"encoding/binary"
"fmt"
"strconv"
"unicode/utf8"
)
// Compare returns -1, 0, or +1 depending on whether a is 'less than', 'equal
// to' or 'greater than' b. The two arguments can only be 'equal' if their
// contents are exactly equal. Furthermore, the empty slice must be 'less than'
// any non-empty slice. Compare is used to compare user keys, such as those
// passed as arguments to the various DB methods, as well as those returned
// from Separator, Successor, and Split.
type Compare func(a, b []byte) int
// Equal returns true if a and b are equivalent. For a given Compare,
// Equal(a,b) must return true iff Compare(a,b) returns zero, that is,
// Equal is a (potentially faster) specialization of Compare.
type Equal func(a, b []byte) bool
// AbbreviatedKey returns a fixed length prefix of a user key such that AbbreviatedKey(a)
// < AbbreviatedKey(b) iff a < b and AbbreviatedKey(a) > AbbreviatedKey(b) iff a > b. If
// AbbreviatedKey(a) == AbbreviatedKey(b) an additional comparison is required to
// determine if the two keys are actually equal.
//
// This helps optimize indexed batch comparisons for cache locality. If a Split
// function is specified, AbbreviatedKey usually returns the first eight bytes
// of the user key prefix in the order that gives the correct ordering.
type AbbreviatedKey func(key []byte) uint64
// FormatKey returns a formatter for the user key.
type FormatKey func(key []byte) fmt.Formatter
// FormatValue returns a formatter for the user value. The key is also
// specified for the value formatter in order to support value formatting that
// is dependent on the key.
type FormatValue func(key, value []byte) fmt.Formatter
// Separator is used to construct SSTable index blocks. A trivial implementation
// is `return a`, but appending fewer bytes leads to smaller SSTables.
//
// Given keys a, b for which Compare(a, b) < 0, Separator returns a key k such
// that:
//
// 1. Compare(a, k) <= 0, and
// 2. Compare(k, b) < 0.
//
// As a special case, b may be nil in which case the second condition is dropped.
//
// For example, if dst, a and b are the []byte equivalents of the strings
// "aqua", "black" and "blue", then the result may be "aquablb".
// Similarly, if the arguments were "aqua", "green" and "", then the result
// may be "aquah".
type Separator func(dst, a, b []byte) []byte
// Successor returns a shortened key given a key a, such that Compare(k, a) >=
// 0. A simple implementation may return a unchanged. The dst parameter may be
// used to store the returned key, though it is valid to pass nil. The returned
// key must be valid to pass to Compare.
type Successor func(dst, a []byte) []byte
// ImmediateSuccessor is invoked with a prefix key ([Split(a) == len(a)]) and
// returns the smallest key that is larger than the given prefix a.
// ImmediateSuccessor must return a prefix key k such that:
//
// Split(k) == len(k) and Compare(k, a) > 0
//
// and there exists no representable k2 such that:
//
// Split(k2) == len(k2) and Compare(k2, a) > 0 and Compare(k2, k) < 0
//
// As an example, an implementation built on the natural byte ordering using
// bytes.Compare could append a `\0` to `a`.
//
// The dst parameter may be used to store the returned key, though it is valid
// to pass nil. The returned key must be valid to pass to Compare.
type ImmediateSuccessor func(dst, a []byte) []byte
// Split returns the length of the prefix of the user key that corresponds to
// the key portion of an MVCC encoding scheme to enable the use of prefix bloom
// filters.
//
// The method will only ever be called with valid MVCC keys, that is, keys that
// the user could potentially store in the database. Pebble does not know which
// keys are MVCC keys and which are not, and may call Split on both MVCC keys
// and non-MVCC keys.
//
// A trivial MVCC scheme is one in which Split() returns len(a). This
// corresponds to assigning a constant version to each key in the database. For
// performance reasons, it is preferable to use a `nil` split in this case.
//
// The returned prefix must have the following properties:
//
// 1. The prefix must be a byte prefix:
//
// bytes.HasPrefix(a, prefix(a))
//
// 2. A key consisting of just a prefix must sort before all other keys with
// that prefix:
//
// Compare(prefix(a), a) < 0 if len(suffix(a)) > 0
//
// 3. Prefixes must be used to order keys before suffixes:
//
// If Compare(a, b) <= 0, then Compare(prefix(a), prefix(b)) <= 0
//
// 4. Suffixes themselves must be valid keys and comparable, respecting the same
// ordering as within a key.
//
// If Compare(prefix(a), prefix(b)) == 0, then Compare(suffix(a), suffix(b)) == Compare(a, b)
type Split func(a []byte) int
// Comparer defines a total ordering over the space of []byte keys: a 'less
// than' relationship.
type Comparer struct {
Compare Compare
Equal Equal
AbbreviatedKey AbbreviatedKey
FormatKey FormatKey
FormatValue FormatValue
Separator Separator
Split Split
Successor Successor
ImmediateSuccessor ImmediateSuccessor
// Name is the name of the comparer.
//
// The Level-DB on-disk format stores the comparer name, and opening a
// database with a different comparer from the one it was created with
// will result in an error.
Name string
}
// DefaultFormatter is the default implementation of user key formatting:
// non-ASCII data is formatted as escaped hexadecimal values.
var DefaultFormatter = func(key []byte) fmt.Formatter {
return FormatBytes(key)
}
// DefaultComparer is the default implementation of the Comparer interface.
// It uses the natural ordering, consistent with bytes.Compare.
var DefaultComparer = &Comparer{
Compare: bytes.Compare,
Equal: bytes.Equal,
AbbreviatedKey: func(key []byte) uint64 {
if len(key) >= 8 {
return binary.BigEndian.Uint64(key)
}
var v uint64
for _, b := range key {
v <<= 8
v |= uint64(b)
}
return v << uint(8*(8-len(key)))
},
FormatKey: DefaultFormatter,
Separator: func(dst, a, b []byte) []byte {
i, n := SharedPrefixLen(a, b), len(dst)
dst = append(dst, a...)
min := len(a)
if min > len(b) {
min = len(b)
}
if i >= min {
// Do not shorten if one string is a prefix of the other.
return dst
}
if a[i] >= b[i] {
// b is smaller than a or a is already the shortest possible.
return dst
}
if i < len(b)-1 || a[i]+1 < b[i] {
i += n
dst[i]++
return dst[:i+1]
}
i += n + 1
for ; i < len(dst); i++ {
if dst[i] != 0xff {
dst[i]++
return dst[:i+1]
}
}
return dst
},
Successor: func(dst, a []byte) (ret []byte) {
for i := 0; i < len(a); i++ {
if a[i] != 0xff {
dst = append(dst, a[:i+1]...)
dst[len(dst)-1]++
return dst
}
}
// a is a run of 0xffs, leave it alone.
return append(dst, a...)
},
ImmediateSuccessor: func(dst, a []byte) (ret []byte) {
return append(append(dst, a...), 0x00)
},
// This name is part of the C++ Level-DB implementation's default file
// format, and should not be changed.
Name: "leveldb.BytewiseComparator",
}
// SharedPrefixLen returns the largest i such that a[:i] equals b[:i].
// This function can be useful in implementing the Comparer interface.
func SharedPrefixLen(a, b []byte) int {
i, n := 0, len(a)
if n > len(b) {
n = len(b)
}
asUint64 := func(c []byte, i int) uint64 {
return binary.LittleEndian.Uint64(c[i:])
}
for i < n-7 && asUint64(a, i) == asUint64(b, i) {
i += 8
}
for i < n && a[i] == b[i] {
i++
}
return i
}
// FormatBytes formats a byte slice using hexadecimal escapes for non-ASCII
// data.
type FormatBytes []byte
const lowerhex = "0123456789abcdef"
// Format implements the fmt.Formatter interface.
func (p FormatBytes) Format(s fmt.State, c rune) {
buf := make([]byte, 0, len(p))
for _, b := range p {
if b < utf8.RuneSelf && strconv.IsPrint(rune(b)) {
buf = append(buf, b)
continue
}
buf = append(buf, `\x`...)
buf = append(buf, lowerhex[b>>4])
buf = append(buf, lowerhex[b&0xF])
}
s.Write(buf)
}

28
vendor/github.com/cockroachdb/pebble/internal/base/error.go generated vendored Normal file
View File

@@ -0,0 +1,28 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "github.com/cockroachdb/errors"
// ErrNotFound means that a get or delete call did not find the requested key.
var ErrNotFound = errors.New("pebble: not found")
// ErrCorruption is a marker to indicate that data in a file (WAL, MANIFEST,
// sstable) isn't in the expected format.
var ErrCorruption = errors.New("pebble: corruption")
// MarkCorruptionError marks given error as a corruption error.
func MarkCorruptionError(err error) error {
if errors.Is(err, ErrCorruption) {
return err
}
return errors.Mark(err, ErrCorruption)
}
// CorruptionErrorf formats according to a format specifier and returns
// the string as an error value that is marked as a corruption error.
func CorruptionErrorf(format string, args ...interface{}) error {
return errors.Mark(errors.Newf(format, args...), ErrCorruption)
}

194
vendor/github.com/cockroachdb/pebble/internal/base/filenames.go generated vendored Normal file
View File

@@ -0,0 +1,194 @@
// Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import (
"fmt"
"strconv"
"strings"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/redact"
)
// FileNum is an internal DB identifier for a file.
type FileNum uint64
// String returns a string representation of the file number.
func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
// DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
// called if the caller can ensure that the FileNum belongs to a physical file
// on disk. These could be manifests, log files, physical sstables on disk, the
// options file, but not virtual sstables.
func (fn FileNum) DiskFileNum() DiskFileNum {
return DiskFileNum{fn}
}
// A DiskFileNum is just a FileNum belonging to a file which exists on disk.
// Note that a FileNum is an internal DB identifier and it could belong to files
// which don't exist on disk. An example would be virtual sstable FileNums.
// Converting a DiskFileNum to a FileNum is always valid, whereas converting a
// FileNum to DiskFileNum may not be valid and care should be taken to prove
// that the FileNum actually exists on disk.
type DiskFileNum struct {
fn FileNum
}
func (dfn DiskFileNum) String() string { return dfn.fn.String() }
// FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
func (dfn DiskFileNum) FileNum() FileNum {
return dfn.fn
}
// FileType enumerates the types of files found in a DB.
type FileType int
// The FileType enumeration.
const (
FileTypeLog FileType = iota
FileTypeLock
FileTypeTable
FileTypeManifest
FileTypeCurrent
FileTypeOptions
FileTypeOldTemp
FileTypeTemp
)
// MakeFilename builds a filename from components.
func MakeFilename(fileType FileType, dfn DiskFileNum) string {
switch fileType {
case FileTypeLog:
return fmt.Sprintf("%s.log", dfn)
case FileTypeLock:
return "LOCK"
case FileTypeTable:
return fmt.Sprintf("%s.sst", dfn)
case FileTypeManifest:
return fmt.Sprintf("MANIFEST-%s", dfn)
case FileTypeCurrent:
return "CURRENT"
case FileTypeOptions:
return fmt.Sprintf("OPTIONS-%s", dfn)
case FileTypeOldTemp:
return fmt.Sprintf("CURRENT.%s.dbtmp", dfn)
case FileTypeTemp:
return fmt.Sprintf("temporary.%s.dbtmp", dfn)
}
panic("unreachable")
}
// MakeFilepath builds a filepath from components.
func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, dfn DiskFileNum) string {
return fs.PathJoin(dirname, MakeFilename(fileType, dfn))
}
// ParseFilename parses the components from a filename.
func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileNum, ok bool) {
filename = fs.PathBase(filename)
switch {
case filename == "CURRENT":
return FileTypeCurrent, DiskFileNum{0}, true
case filename == "LOCK":
return FileTypeLock, DiskFileNum{0}, true
case strings.HasPrefix(filename, "MANIFEST-"):
dfn, ok = parseDiskFileNum(filename[len("MANIFEST-"):])
if !ok {
break
}
return FileTypeManifest, dfn, true
case strings.HasPrefix(filename, "OPTIONS-"):
dfn, ok = parseDiskFileNum(filename[len("OPTIONS-"):])
if !ok {
break
}
return FileTypeOptions, dfn, ok
case strings.HasPrefix(filename, "CURRENT.") && strings.HasSuffix(filename, ".dbtmp"):
s := strings.TrimSuffix(filename[len("CURRENT."):], ".dbtmp")
dfn, ok = parseDiskFileNum(s)
if !ok {
break
}
return FileTypeOldTemp, dfn, ok
case strings.HasPrefix(filename, "temporary.") && strings.HasSuffix(filename, ".dbtmp"):
s := strings.TrimSuffix(filename[len("temporary."):], ".dbtmp")
dfn, ok = parseDiskFileNum(s)
if !ok {
break
}
return FileTypeTemp, dfn, ok
default:
i := strings.IndexByte(filename, '.')
if i < 0 {
break
}
dfn, ok = parseDiskFileNum(filename[:i])
if !ok {
break
}
switch filename[i+1:] {
case "sst":
return FileTypeTable, dfn, true
case "log":
return FileTypeLog, dfn, true
}
}
return 0, dfn, false
}
func parseDiskFileNum(s string) (dfn DiskFileNum, ok bool) {
u, err := strconv.ParseUint(s, 10, 64)
if err != nil {
return dfn, false
}
return DiskFileNum{FileNum(u)}, true
}
// A Fataler fatals a process with a message when called.
type Fataler interface {
Fatalf(format string, args ...interface{})
}
// MustExist checks if err is an error indicating a file does not exist.
// If it is, it lists the containing directory's files to annotate the error
// with counts of the various types of files and invokes the provided fataler.
// See cockroachdb/cockroach#56490.
func MustExist(fs vfs.FS, filename string, fataler Fataler, err error) {
if err == nil || !oserror.IsNotExist(err) {
return
}
ls, lsErr := fs.List(fs.PathDir(filename))
if lsErr != nil {
// TODO(jackson): if oserror.IsNotExist(lsErr), the the data directory
// doesn't exist anymore. Another process likely deleted it before
// killing the process. We want to fatal the process, but without
// triggering error reporting like Sentry.
fataler.Fatalf("%s:\norig err: %s\nlist err: %s", redact.Safe(fs.PathBase(filename)), err, lsErr)
}
var total, unknown, tables, logs, manifests int
total = len(ls)
for _, f := range ls {
typ, _, ok := ParseFilename(fs, f)
if !ok {
unknown++
continue
}
switch typ {
case FileTypeTable:
tables++
case FileTypeLog:
logs++
case FileTypeManifest:
manifests++
}
}
fataler.Fatalf("%s:\n%s\ndirectory contains %d files, %d unknown, %d tables, %d logs, %d manifests",
fs.PathBase(filename), err, total, unknown, tables, logs, manifests)
}

494
vendor/github.com/cockroachdb/pebble/internal/base/internal.go generated vendored Normal file
View File

@@ -0,0 +1,494 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base // import "github.com/cockroachdb/pebble/internal/base"
import (
"encoding/binary"
"fmt"
"strconv"
"strings"
)
const (
// SeqNumZero is the zero sequence number, set by compactions if they can
// guarantee there are no keys underneath an internal key.
SeqNumZero = uint64(0)
// SeqNumStart is the first sequence number assigned to a key. Sequence
// numbers 1-9 are reserved for potential future use.
SeqNumStart = uint64(10)
)
// InternalKeyKind enumerates the kind of key: a deletion tombstone, a set
// value, a merged value, etc.
type InternalKeyKind uint8
// These constants are part of the file format, and should not be changed.
const (
InternalKeyKindDelete InternalKeyKind = 0
InternalKeyKindSet InternalKeyKind = 1
InternalKeyKindMerge InternalKeyKind = 2
InternalKeyKindLogData InternalKeyKind = 3
//InternalKeyKindColumnFamilyDeletion InternalKeyKind = 4
//InternalKeyKindColumnFamilyValue InternalKeyKind = 5
//InternalKeyKindColumnFamilyMerge InternalKeyKind = 6
// InternalKeyKindSingleDelete (SINGLEDEL) is a performance optimization
// solely for compactions (to reduce write amp and space amp). Readers other
// than compactions should treat SINGLEDEL as equivalent to a DEL.
// Historically, it was simpler for readers other than compactions to treat
// SINGLEDEL as equivalent to DEL, but as of the introduction of
// InternalKeyKindSSTableInternalObsoleteBit, this is also necessary for
// correctness.
InternalKeyKindSingleDelete InternalKeyKind = 7
//InternalKeyKindColumnFamilySingleDelete InternalKeyKind = 8
//InternalKeyKindBeginPrepareXID InternalKeyKind = 9
//InternalKeyKindEndPrepareXID InternalKeyKind = 10
//InternalKeyKindCommitXID InternalKeyKind = 11
//InternalKeyKindRollbackXID InternalKeyKind = 12
//InternalKeyKindNoop InternalKeyKind = 13
//InternalKeyKindColumnFamilyRangeDelete InternalKeyKind = 14
InternalKeyKindRangeDelete InternalKeyKind = 15
//InternalKeyKindColumnFamilyBlobIndex InternalKeyKind = 16
//InternalKeyKindBlobIndex InternalKeyKind = 17
// InternalKeyKindSeparator is a key used for separator / successor keys
// written to sstable block indexes.
//
// NOTE: the RocksDB value has been repurposed. This was done to ensure that
// keys written to block indexes with value "17" (when 17 happened to be the
// max value, and InternalKeyKindMax was therefore set to 17), remain stable
// when new key kinds are supported in Pebble.
InternalKeyKindSeparator InternalKeyKind = 17
// InternalKeyKindSetWithDelete keys are SET keys that have met with a
// DELETE or SINGLEDEL key in a prior compaction. This key kind is
// specific to Pebble. See
// https://github.com/cockroachdb/pebble/issues/1255.
InternalKeyKindSetWithDelete InternalKeyKind = 18
// InternalKeyKindRangeKeyDelete removes all range keys within a key range.
// See the internal/rangekey package for more details.
InternalKeyKindRangeKeyDelete InternalKeyKind = 19
// InternalKeyKindRangeKeySet and InternalKeyKindRangeUnset represent
// keys that set and unset values associated with ranges of key
// space. See the internal/rangekey package for more details.
InternalKeyKindRangeKeyUnset InternalKeyKind = 20
InternalKeyKindRangeKeySet InternalKeyKind = 21
// InternalKeyKindIngestSST is used to distinguish a batch that corresponds to
// the WAL entry for ingested sstables that are added to the flushable
// queue. This InternalKeyKind cannot appear, amongst other key kinds in a
// batch, or in an sstable.
InternalKeyKindIngestSST InternalKeyKind = 22
// InternalKeyKindDeleteSized keys behave identically to
// InternalKeyKindDelete keys, except that they hold an associated uint64
// value indicating the (len(key)+len(value)) of the shadowed entry the
// tombstone is expected to delete. This value is used to inform compaction
// heuristics, but is not required to be accurate for correctness.
InternalKeyKindDeleteSized InternalKeyKind = 23
// This maximum value isn't part of the file format. Future extensions may
// increase this value.
//
// When constructing an internal key to pass to DB.Seek{GE,LE},
// internalKeyComparer sorts decreasing by kind (after sorting increasing by
// user key and decreasing by sequence number). Thus, use InternalKeyKindMax,
// which sorts 'less than or equal to' any other valid internalKeyKind, when
// searching for any kind of internal key formed by a certain user key and
// seqNum.
InternalKeyKindMax InternalKeyKind = 23
// Internal to the sstable format. Not exposed by any sstable iterator.
// Declared here to prevent definition of valid key kinds that set this bit.
InternalKeyKindSSTableInternalObsoleteBit InternalKeyKind = 64
InternalKeyKindSSTableInternalObsoleteMask InternalKeyKind = 191
// InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a
// zero sequence number.
InternalKeyZeroSeqnumMaxTrailer = uint64(255)
// A marker for an invalid key.
InternalKeyKindInvalid InternalKeyKind = InternalKeyKindSSTableInternalObsoleteMask
// InternalKeySeqNumBatch is a bit that is set on batch sequence numbers
// which prevents those entries from being excluded from iteration.
InternalKeySeqNumBatch = uint64(1 << 55)
// InternalKeySeqNumMax is the largest valid sequence number.
InternalKeySeqNumMax = uint64(1<<56 - 1)
// InternalKeyRangeDeleteSentinel is the marker for a range delete sentinel
// key. This sequence number and kind are used for the upper stable boundary
// when a range deletion tombstone is the largest key in an sstable. This is
// necessary because sstable boundaries are inclusive, while the end key of a
// range deletion tombstone is exclusive.
InternalKeyRangeDeleteSentinel = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeDelete)
// InternalKeyBoundaryRangeKey is the marker for a range key boundary. This
// sequence number and kind are used during interleaved range key and point
// iteration to allow an iterator to stop at range key start keys where
// there exists no point key.
InternalKeyBoundaryRangeKey = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeKeySet)
)
// Assert InternalKeyKindSSTableInternalObsoleteBit > InternalKeyKindMax
const _ = uint(InternalKeyKindSSTableInternalObsoleteBit - InternalKeyKindMax - 1)
var internalKeyKindNames = []string{
InternalKeyKindDelete: "DEL",
InternalKeyKindSet: "SET",
InternalKeyKindMerge: "MERGE",
InternalKeyKindLogData: "LOGDATA",
InternalKeyKindSingleDelete: "SINGLEDEL",
InternalKeyKindRangeDelete: "RANGEDEL",
InternalKeyKindSeparator: "SEPARATOR",
InternalKeyKindSetWithDelete: "SETWITHDEL",
InternalKeyKindRangeKeySet: "RANGEKEYSET",
InternalKeyKindRangeKeyUnset: "RANGEKEYUNSET",
InternalKeyKindRangeKeyDelete: "RANGEKEYDEL",
InternalKeyKindIngestSST: "INGESTSST",
InternalKeyKindDeleteSized: "DELSIZED",
InternalKeyKindInvalid: "INVALID",
}
func (k InternalKeyKind) String() string {
if int(k) < len(internalKeyKindNames) {
return internalKeyKindNames[k]
}
return fmt.Sprintf("UNKNOWN:%d", k)
}
// InternalKey is a key used for the in-memory and on-disk partial DBs that
// make up a pebble DB.
//
// It consists of the user key (as given by the code that uses package pebble)
// followed by 8-bytes of metadata:
// - 1 byte for the type of internal key: delete or set,
// - 7 bytes for a uint56 sequence number, in little-endian format.
type InternalKey struct {
UserKey []byte
Trailer uint64
}
// InvalidInternalKey is an invalid internal key for which Valid() will return
// false.
var InvalidInternalKey = MakeInternalKey(nil, 0, InternalKeyKindInvalid)
// MakeInternalKey constructs an internal key from a specified user key,
// sequence number and kind.
func MakeInternalKey(userKey []byte, seqNum uint64, kind InternalKeyKind) InternalKey {
return InternalKey{
UserKey: userKey,
Trailer: (seqNum << 8) | uint64(kind),
}
}
// MakeTrailer constructs an internal key trailer from the specified sequence
// number and kind.
func MakeTrailer(seqNum uint64, kind InternalKeyKind) uint64 {
return (seqNum << 8) | uint64(kind)
}
// MakeSearchKey constructs an internal key that is appropriate for searching
// for a the specified user key. The search key contain the maximal sequence
// number and kind ensuring that it sorts before any other internal keys for
// the same user key.
func MakeSearchKey(userKey []byte) InternalKey {
return InternalKey{
UserKey: userKey,
Trailer: (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindMax),
}
}
// MakeRangeDeleteSentinelKey constructs an internal key that is a range
// deletion sentinel key, used as the upper boundary for an sstable when a
// range deletion is the largest key in an sstable.
func MakeRangeDeleteSentinelKey(userKey []byte) InternalKey {
return InternalKey{
UserKey: userKey,
Trailer: InternalKeyRangeDeleteSentinel,
}
}
// MakeExclusiveSentinelKey constructs an internal key that is an
// exclusive sentinel key, used as the upper boundary for an sstable
// when a ranged key is the largest key in an sstable.
func MakeExclusiveSentinelKey(kind InternalKeyKind, userKey []byte) InternalKey {
return InternalKey{
UserKey: userKey,
Trailer: (InternalKeySeqNumMax << 8) | uint64(kind),
}
}
var kindsMap = map[string]InternalKeyKind{
"DEL": InternalKeyKindDelete,
"SINGLEDEL": InternalKeyKindSingleDelete,
"RANGEDEL": InternalKeyKindRangeDelete,
"LOGDATA": InternalKeyKindLogData,
"SET": InternalKeyKindSet,
"MERGE": InternalKeyKindMerge,
"INVALID": InternalKeyKindInvalid,
"SEPARATOR": InternalKeyKindSeparator,
"SETWITHDEL": InternalKeyKindSetWithDelete,
"RANGEKEYSET": InternalKeyKindRangeKeySet,
"RANGEKEYUNSET": InternalKeyKindRangeKeyUnset,
"RANGEKEYDEL": InternalKeyKindRangeKeyDelete,
"INGESTSST": InternalKeyKindIngestSST,
"DELSIZED": InternalKeyKindDeleteSized,
}
// ParseInternalKey parses the string representation of an internal key. The
// format is <user-key>.<kind>.<seq-num>. If the seq-num starts with a "b" it
// is marked as a batch-seq-num (i.e. the InternalKeySeqNumBatch bit is set).
func ParseInternalKey(s string) InternalKey {
x := strings.Split(s, ".")
ukey := x[0]
kind, ok := kindsMap[x[1]]
if !ok {
panic(fmt.Sprintf("unknown kind: %q", x[1]))
}
j := 0
if x[2][0] == 'b' {
j = 1
}
seqNum, _ := strconv.ParseUint(x[2][j:], 10, 64)
if x[2][0] == 'b' {
seqNum |= InternalKeySeqNumBatch
}
return MakeInternalKey([]byte(ukey), seqNum, kind)
}
// ParseKind parses the string representation of an internal key kind.
func ParseKind(s string) InternalKeyKind {
kind, ok := kindsMap[s]
if !ok {
panic(fmt.Sprintf("unknown kind: %q", s))
}
return kind
}
// InternalTrailerLen is the number of bytes used to encode InternalKey.Trailer.
const InternalTrailerLen = 8
// DecodeInternalKey decodes an encoded internal key. See InternalKey.Encode().
func DecodeInternalKey(encodedKey []byte) InternalKey {
n := len(encodedKey) - InternalTrailerLen
var trailer uint64
if n >= 0 {
trailer = binary.LittleEndian.Uint64(encodedKey[n:])
encodedKey = encodedKey[:n:n]
} else {
trailer = uint64(InternalKeyKindInvalid)
encodedKey = nil
}
return InternalKey{
UserKey: encodedKey,
Trailer: trailer,
}
}
// InternalCompare compares two internal keys using the specified comparison
// function. For equal user keys, internal keys compare in descending sequence
// number order. For equal user keys and sequence numbers, internal keys
// compare in descending kind order (this may happen in practice among range
// keys).
func InternalCompare(userCmp Compare, a, b InternalKey) int {
if x := userCmp(a.UserKey, b.UserKey); x != 0 {
return x
}
if a.Trailer > b.Trailer {
return -1
}
if a.Trailer < b.Trailer {
return 1
}
return 0
}
// Encode encodes the receiver into the buffer. The buffer must be large enough
// to hold the encoded data. See InternalKey.Size().
func (k InternalKey) Encode(buf []byte) {
i := copy(buf, k.UserKey)
binary.LittleEndian.PutUint64(buf[i:], k.Trailer)
}
// EncodeTrailer returns the trailer encoded to an 8-byte array.
func (k InternalKey) EncodeTrailer() [8]byte {
var buf [8]byte
binary.LittleEndian.PutUint64(buf[:], k.Trailer)
return buf
}
// Separator returns a separator key such that k <= x && x < other, where less
// than is consistent with the Compare function. The buf parameter may be used
// to store the returned InternalKey.UserKey, though it is valid to pass a
// nil. See the Separator type for details on separator keys.
func (k InternalKey) Separator(
cmp Compare, sep Separator, buf []byte, other InternalKey,
) InternalKey {
buf = sep(buf, k.UserKey, other.UserKey)
if len(buf) <= len(k.UserKey) && cmp(k.UserKey, buf) < 0 {
// The separator user key is physically shorter than k.UserKey (if it is
// longer, we'll continue to use "k"), but logically after. Tack on the max
// sequence number to the shortened user key. Note that we could tack on
// any sequence number and kind here to create a valid separator key. We
// use the max sequence number to match the behavior of LevelDB and
// RocksDB.
return MakeInternalKey(buf, InternalKeySeqNumMax, InternalKeyKindSeparator)
}
return k
}
// Successor returns a successor key such that k <= x. A simple implementation
// may return k unchanged. The buf parameter may be used to store the returned
// InternalKey.UserKey, though it is valid to pass a nil.
func (k InternalKey) Successor(cmp Compare, succ Successor, buf []byte) InternalKey {
buf = succ(buf, k.UserKey)
if len(buf) <= len(k.UserKey) && cmp(k.UserKey, buf) < 0 {
// The successor user key is physically shorter that k.UserKey (if it is
// longer, we'll continue to use "k"), but logically after. Tack on the max
// sequence number to the shortened user key. Note that we could tack on
// any sequence number and kind here to create a valid separator key. We
// use the max sequence number to match the behavior of LevelDB and
// RocksDB.
return MakeInternalKey(buf, InternalKeySeqNumMax, InternalKeyKindSeparator)
}
return k
}
// Size returns the encoded size of the key.
func (k InternalKey) Size() int {
return len(k.UserKey) + 8
}
// SetSeqNum sets the sequence number component of the key.
func (k *InternalKey) SetSeqNum(seqNum uint64) {
k.Trailer = (seqNum << 8) | (k.Trailer & 0xff)
}
// SeqNum returns the sequence number component of the key.
func (k InternalKey) SeqNum() uint64 {
return k.Trailer >> 8
}
// Visible returns true if the key is visible at the specified snapshot
// sequence number.
func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool {
return Visible(k.SeqNum(), snapshot, batchSnapshot)
}
// Visible returns true if a key with the provided sequence number is visible at
// the specified snapshot sequence numbers.
func Visible(seqNum uint64, snapshot, batchSnapshot uint64) bool {
// There are two snapshot sequence numbers, one for committed keys and one
// for batch keys. If a seqNum is less than `snapshot`, then seqNum
// corresponds to a committed key that is visible. If seqNum has its batch
// bit set, then seqNum corresponds to an uncommitted batch key. Its
// visible if its snapshot is less than batchSnapshot.
//
// There's one complication. The maximal sequence number
// (`InternalKeySeqNumMax`) is used across Pebble for exclusive sentinel
// keys and other purposes. The maximal sequence number has its batch bit
// set, but it can never be < `batchSnapshot`, since there is no expressible
// larger snapshot. We dictate that the maximal sequence number is always
// visible.
return seqNum < snapshot ||
((seqNum&InternalKeySeqNumBatch) != 0 && seqNum < batchSnapshot) ||
seqNum == InternalKeySeqNumMax
}
// SetKind sets the kind component of the key.
func (k *InternalKey) SetKind(kind InternalKeyKind) {
k.Trailer = (k.Trailer &^ 0xff) | uint64(kind)
}
// Kind returns the kind component of the key.
func (k InternalKey) Kind() InternalKeyKind {
return TrailerKind(k.Trailer)
}
// TrailerKind returns the key kind of the key trailer.
func TrailerKind(trailer uint64) InternalKeyKind {
return InternalKeyKind(trailer & 0xff)
}
// Valid returns true if the key has a valid kind.
func (k InternalKey) Valid() bool {
return k.Kind() <= InternalKeyKindMax
}
// Clone clones the storage for the UserKey component of the key.
func (k InternalKey) Clone() InternalKey {
if len(k.UserKey) == 0 {
return k
}
return InternalKey{
UserKey: append([]byte(nil), k.UserKey...),
Trailer: k.Trailer,
}
}
// CopyFrom converts this InternalKey into a clone of the passed-in InternalKey,
// reusing any space already used for the current UserKey.
func (k *InternalKey) CopyFrom(k2 InternalKey) {
k.UserKey = append(k.UserKey[:0], k2.UserKey...)
k.Trailer = k2.Trailer
}
// String returns a string representation of the key.
func (k InternalKey) String() string {
return fmt.Sprintf("%s#%d,%d", FormatBytes(k.UserKey), k.SeqNum(), k.Kind())
}
// Pretty returns a formatter for the key.
func (k InternalKey) Pretty(f FormatKey) fmt.Formatter {
return prettyInternalKey{k, f}
}
// IsExclusiveSentinel returns whether this internal key excludes point keys
// with the same user key if used as an end boundary. See the comment on
// InternalKeyRangeDeletionSentinel.
func (k InternalKey) IsExclusiveSentinel() bool {
switch kind := k.Kind(); kind {
case InternalKeyKindRangeDelete:
return k.Trailer == InternalKeyRangeDeleteSentinel
case InternalKeyKindRangeKeyDelete, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet:
return (k.Trailer >> 8) == InternalKeySeqNumMax
default:
return false
}
}
type prettyInternalKey struct {
InternalKey
formatKey FormatKey
}
func (k prettyInternalKey) Format(s fmt.State, c rune) {
if seqNum := k.SeqNum(); seqNum == InternalKeySeqNumMax {
fmt.Fprintf(s, "%s#inf,%s", k.formatKey(k.UserKey), k.Kind())
} else {
fmt.Fprintf(s, "%s#%d,%s", k.formatKey(k.UserKey), k.SeqNum(), k.Kind())
}
}
// ParsePrettyInternalKey parses the pretty string representation of an
// internal key. The format is <user-key>#<seq-num>,<kind>.
func ParsePrettyInternalKey(s string) InternalKey {
x := strings.FieldsFunc(s, func(c rune) bool { return c == '#' || c == ',' })
ukey := x[0]
kind, ok := kindsMap[x[2]]
if !ok {
panic(fmt.Sprintf("unknown kind: %q", x[2]))
}
var seqNum uint64
if x[1] == "max" || x[1] == "inf" {
seqNum = InternalKeySeqNumMax
} else {
seqNum, _ = strconv.ParseUint(x[1], 10, 64)
}
return MakeInternalKey([]byte(ukey), seqNum, kind)
}

409
vendor/github.com/cockroachdb/pebble/internal/base/iterator.go generated vendored Normal file
View File

@@ -0,0 +1,409 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import (
"fmt"
"time"
)
// InternalIterator iterates over a DB's key/value pairs in key order. Unlike
// the Iterator interface, the returned keys are InternalKeys composed of the
// user-key, a sequence number and a key kind. In forward iteration, key/value
// pairs for identical user-keys are returned in descending sequence order. In
// reverse iteration, key/value pairs for identical user-keys are returned in
// ascending sequence order.
//
// InternalIterators provide 5 absolute positioning methods and 2 relative
// positioning methods. The absolute positioning methods are:
//
// - SeekGE
// - SeekPrefixGE
// - SeekLT
// - First
// - Last
//
// The relative positioning methods are:
//
// - Next
// - Prev
//
// The relative positioning methods can be used in conjunction with any of the
// absolute positioning methods with one exception: SeekPrefixGE does not
// support reverse iteration via Prev. It is undefined to call relative
// positioning methods without ever calling an absolute positioning method.
//
// InternalIterators can optionally implement a prefix iteration mode. This
// mode is entered by calling SeekPrefixGE and exited by any other absolute
// positioning method (SeekGE, SeekLT, First, Last). When in prefix iteration
// mode, a call to Next will advance to the next key which has the same
// "prefix" as the one supplied to SeekPrefixGE. Note that "prefix" in this
// context is not a strict byte prefix, but defined by byte equality for the
// result of the Comparer.Split method. An InternalIterator is not required to
// support prefix iteration mode, and can implement SeekPrefixGE by forwarding
// to SeekGE. When the iteration prefix is exhausted, it is not valid to call
// Next on an internal iterator that's already returned (nil,nilv) or a key
// beyond the prefix.
//
// Bounds, [lower, upper), can be set on iterators, either using the SetBounds()
// function in the interface, or in implementation specific ways during iterator
// creation. The forward positioning routines (SeekGE, First, and Next) only
// check the upper bound. The reverse positioning routines (SeekLT, Last, and
// Prev) only check the lower bound. It is up to the caller to ensure that the
// forward positioning routines respect the lower bound and the reverse
// positioning routines respect the upper bound (i.e. calling SeekGE instead of
// First if there is a lower bound, and SeekLT instead of Last if there is an
// upper bound). This imposition is done in order to elevate that enforcement to
// the caller (generally pebble.Iterator or pebble.mergingIter) rather than
// having it duplicated in every InternalIterator implementation.
//
// Additionally, the caller needs to ensure that SeekGE/SeekPrefixGE are not
// called with a key > the upper bound, and SeekLT is not called with a key <
// the lower bound. InternalIterator implementations are required to respect
// the iterator bounds, never returning records outside of the bounds with one
// exception: an iterator may generate synthetic RANGEDEL marker records. See
// levelIter.syntheticBoundary for the sole existing example of this behavior.
// Specifically, levelIter can return synthetic keys whose user key is equal to
// the lower/upper bound.
//
// The bounds provided to an internal iterator must remain valid until a
// subsequent call to SetBounds has returned. This requirement exists so that
// iterator implementations may compare old and new bounds to apply low-level
// optimizations. The pebble.Iterator satisfies this requirement by maintaining
// two bound buffers and switching between them.
//
// An iterator must be closed after use, but it is not necessary to read an
// iterator until exhaustion.
//
// An iterator is not goroutine-safe, but it is safe to use multiple iterators
// concurrently, either in separate goroutines or switching between the
// iterators in a single goroutine.
//
// It is also safe to use an iterator concurrently with modifying its
// underlying DB, if that DB permits modification. However, the resultant
// key/value pairs are not guaranteed to be a consistent snapshot of that DB
// at a particular point in time.
//
// InternalIterators accumulate errors encountered during operation, exposing
// them through the Error method. All of the absolute positioning methods
// reset any accumulated error before positioning. Relative positioning
// methods return without advancing if the iterator has accumulated an error.
//
// nilv == shorthand for LazyValue{}, which represents a nil value.
type InternalIterator interface {
// SeekGE moves the iterator to the first key/value pair whose key is greater
// than or equal to the given key. Returns the key and value if the iterator
// is pointing at a valid entry, and (nil, nilv) otherwise. Note that SeekGE
// only checks the upper bound. It is up to the caller to ensure that key
// is greater than or equal to the lower bound.
SeekGE(key []byte, flags SeekGEFlags) (*InternalKey, LazyValue)
// SeekPrefixGE moves the iterator to the first key/value pair whose key is
// greater than or equal to the given key. Returns the key and value if the
// iterator is pointing at a valid entry, and (nil, nilv) otherwise. Note that
// SeekPrefixGE only checks the upper bound. It is up to the caller to ensure
// that key is greater than or equal to the lower bound.
//
// The prefix argument is used by some InternalIterator implementations (e.g.
// sstable.Reader) to avoid expensive operations. A user-defined Split
// function must be supplied to the Comparer for the DB. The supplied prefix
// will be the prefix of the given key returned by that Split function. If
// the iterator is able to determine that no key with the prefix exists, it
// can return (nil,nilv). Unlike SeekGE, this is not an indication that
// iteration is exhausted.
//
// Note that the iterator may return keys not matching the prefix. It is up
// to the caller to check if the prefix matches.
//
// Calling SeekPrefixGE places the receiver into prefix iteration mode. Once
// in this mode, reverse iteration may not be supported and will return an
// error. Note that pebble/Iterator.SeekPrefixGE has this same restriction on
// not supporting reverse iteration in prefix iteration mode until a
// different positioning routine (SeekGE, SeekLT, First or Last) switches the
// iterator out of prefix iteration.
SeekPrefixGE(prefix, key []byte, flags SeekGEFlags) (*InternalKey, LazyValue)
// SeekLT moves the iterator to the last key/value pair whose key is less
// than the given key. Returns the key and value if the iterator is pointing
// at a valid entry, and (nil, nilv) otherwise. Note that SeekLT only checks
// the lower bound. It is up to the caller to ensure that key is less than
// the upper bound.
SeekLT(key []byte, flags SeekLTFlags) (*InternalKey, LazyValue)
// First moves the iterator the the first key/value pair. Returns the key and
// value if the iterator is pointing at a valid entry, and (nil, nilv)
// otherwise. Note that First only checks the upper bound. It is up to the
// caller to ensure that First() is not called when there is a lower bound,
// and instead call SeekGE(lower).
First() (*InternalKey, LazyValue)
// Last moves the iterator the the last key/value pair. Returns the key and
// value if the iterator is pointing at a valid entry, and (nil, nilv)
// otherwise. Note that Last only checks the lower bound. It is up to the
// caller to ensure that Last() is not called when there is an upper bound,
// and instead call SeekLT(upper).
Last() (*InternalKey, LazyValue)
// Next moves the iterator to the next key/value pair. Returns the key and
// value if the iterator is pointing at a valid entry, and (nil, nilv)
// otherwise. Note that Next only checks the upper bound. It is up to the
// caller to ensure that key is greater than or equal to the lower bound.
//
// It is valid to call Next when the iterator is positioned before the first
// key/value pair due to either a prior call to SeekLT or Prev which returned
// (nil, nilv). It is not allowed to call Next when the previous call to SeekGE,
// SeekPrefixGE or Next returned (nil, nilv).
Next() (*InternalKey, LazyValue)
// NextPrefix moves the iterator to the next key/value pair with a different
// prefix than the key at the current iterator position. Returns the key and
// value if the iterator is pointing at a valid entry, and (nil, nil)
// otherwise. Note that NextPrefix only checks the upper bound. It is up to
// the caller to ensure that key is greater than or equal to the lower
// bound.
//
// NextPrefix is passed the immediate successor to the current prefix key. A
// valid implementation of NextPrefix is to call SeekGE with succKey.
//
// It is not allowed to call NextPrefix when the previous call was a reverse
// positioning operation or a call to a forward positioning method that
// returned (nil, nilv). It is also not allowed to call NextPrefix when the
// iterator is in prefix iteration mode.
NextPrefix(succKey []byte) (*InternalKey, LazyValue)
// Prev moves the iterator to the previous key/value pair. Returns the key
// and value if the iterator is pointing at a valid entry, and (nil, nilv)
// otherwise. Note that Prev only checks the lower bound. It is up to the
// caller to ensure that key is less than the upper bound.
//
// It is valid to call Prev when the iterator is positioned after the last
// key/value pair due to either a prior call to SeekGE or Next which returned
// (nil, nilv). It is not allowed to call Prev when the previous call to SeekLT
// or Prev returned (nil, nilv).
Prev() (*InternalKey, LazyValue)
// Error returns any accumulated error. It may not include errors returned
// to the client when calling LazyValue.Value().
Error() error
// Close closes the iterator and returns any accumulated error. Exhausting
// all the key/value pairs in a table is not considered to be an error.
// It is valid to call Close multiple times. Other methods should not be
// called after the iterator has been closed.
Close() error
// SetBounds sets the lower and upper bounds for the iterator. Note that the
// result of Next and Prev will be undefined until the iterator has been
// repositioned with SeekGE, SeekPrefixGE, SeekLT, First, or Last.
//
// The bounds provided must remain valid until a subsequent call to
// SetBounds has returned. This requirement exists so that iterator
// implementations may compare old and new bounds to apply low-level
// optimizations.
SetBounds(lower, upper []byte)
fmt.Stringer
}
// SeekGEFlags holds flags that may configure the behavior of a forward seek.
// Not all flags are relevant to all iterators.
type SeekGEFlags uint8
const (
seekGEFlagTrySeekUsingNext uint8 = iota
seekGEFlagRelativeSeek
seekGEFlagBatchJustRefreshed
)
// SeekGEFlagsNone is the default value of SeekGEFlags, with all flags disabled.
const SeekGEFlagsNone = SeekGEFlags(0)
// TrySeekUsingNext indicates whether a performance optimization was enabled
// by a caller, indicating the caller has not done any action to move this
// iterator beyond the first key that would be found if this iterator were to
// honestly do the intended seek. For example, say the caller did a
// SeekGE(k1...), followed by SeekGE(k2...) where k1 <= k2, without any
// intermediate positioning calls. The caller can safely specify true for this
// parameter in the second call. As another example, say the caller did do one
// call to Next between the two Seek calls, and k1 < k2. Again, the caller can
// safely specify a true value for this parameter. Note that a false value is
// always safe. The callee is free to ignore the true value if its
// implementation does not permit this optimization.
//
// We make the caller do this determination since a string comparison of k1, k2
// is not necessarily cheap, and there may be many iterators in the iterator
// stack. Doing it once at the root of the iterator stack is cheaper.
//
// This optimization could also be applied to SeekLT (where it would be
// trySeekUsingPrev). We currently only do it for SeekPrefixGE and SeekGE
// because this is where this optimization helps the performance of CockroachDB.
// The SeekLT cases in CockroachDB are typically accompanied with bounds that
// change between seek calls, and is optimized inside certain iterator
// implementations, like singleLevelIterator, without any extra parameter
// passing (though the same amortization of string comparisons could be done to
// improve that optimization, by making the root of the iterator stack do it).
func (s SeekGEFlags) TrySeekUsingNext() bool { return (s & (1 << seekGEFlagTrySeekUsingNext)) != 0 }
// RelativeSeek is set when in the course of a forward positioning operation, a
// higher-level iterator seeks a lower-level iterator to a larger key than the
// one at the current iterator position.
//
// Concretely, this occurs when the merging iterator observes a range deletion
// covering the key at a level's current position, and the merging iterator
// seeks the level to the range deletion's end key. During lazy-combined
// iteration, this flag signals to the level iterator that the seek is NOT an
// absolute-positioning operation from the perspective of the pebble.Iterator,
// and the level iterator must look for range keys in tables between the current
// iterator position and the new seeked position.
func (s SeekGEFlags) RelativeSeek() bool { return (s & (1 << seekGEFlagRelativeSeek)) != 0 }
// BatchJustRefreshed is set by Seek[Prefix]GE when an iterator's view of an
// indexed batch was just refreshed. It serves as a signal to the batch iterator
// to ignore the TrySeekUsingNext optimization, because the external knowledge
// imparted by the TrySeekUsingNext flag does not apply to the batch iterator's
// position. See (pebble.Iterator).batchJustRefreshed.
func (s SeekGEFlags) BatchJustRefreshed() bool { return (s & (1 << seekGEFlagBatchJustRefreshed)) != 0 }
// EnableTrySeekUsingNext returns the provided flags with the
// try-seek-using-next optimization enabled. See TrySeekUsingNext for an
// explanation of this optimization.
func (s SeekGEFlags) EnableTrySeekUsingNext() SeekGEFlags {
return s | (1 << seekGEFlagTrySeekUsingNext)
}
// DisableTrySeekUsingNext returns the provided flags with the
// try-seek-using-next optimization disabled.
func (s SeekGEFlags) DisableTrySeekUsingNext() SeekGEFlags {
return s &^ (1 << seekGEFlagTrySeekUsingNext)
}
// EnableRelativeSeek returns the provided flags with the relative-seek flag
// enabled. See RelativeSeek for an explanation of this flag's use.
func (s SeekGEFlags) EnableRelativeSeek() SeekGEFlags {
return s | (1 << seekGEFlagRelativeSeek)
}
// DisableRelativeSeek returns the provided flags with the relative-seek flag
// disabled.
func (s SeekGEFlags) DisableRelativeSeek() SeekGEFlags {
return s &^ (1 << seekGEFlagRelativeSeek)
}
// EnableBatchJustRefreshed returns the provided flags with the
// batch-just-refreshed bit set. See BatchJustRefreshed for an explanation of
// this flag.
func (s SeekGEFlags) EnableBatchJustRefreshed() SeekGEFlags {
return s | (1 << seekGEFlagBatchJustRefreshed)
}
// DisableBatchJustRefreshed returns the provided flags with the
// batch-just-refreshed bit unset.
func (s SeekGEFlags) DisableBatchJustRefreshed() SeekGEFlags {
return s &^ (1 << seekGEFlagBatchJustRefreshed)
}
// SeekLTFlags holds flags that may configure the behavior of a reverse seek.
// Not all flags are relevant to all iterators.
type SeekLTFlags uint8
const (
seekLTFlagRelativeSeek uint8 = iota
)
// SeekLTFlagsNone is the default value of SeekLTFlags, with all flags disabled.
const SeekLTFlagsNone = SeekLTFlags(0)
// RelativeSeek is set when in the course of a reverse positioning operation, a
// higher-level iterator seeks a lower-level iterator to a smaller key than the
// one at the current iterator position.
//
// Concretely, this occurs when the merging iterator observes a range deletion
// covering the key at a level's current position, and the merging iterator
// seeks the level to the range deletion's start key. During lazy-combined
// iteration, this flag signals to the level iterator that the seek is NOT an
// absolute-positioning operation from the perspective of the pebble.Iterator,
// and the level iterator must look for range keys in tables between the current
// iterator position and the new seeked position.
func (s SeekLTFlags) RelativeSeek() bool { return s&(1<<seekLTFlagRelativeSeek) != 0 }
// EnableRelativeSeek returns the provided flags with the relative-seek flag
// enabled. See RelativeSeek for an explanation of this flag's use.
func (s SeekLTFlags) EnableRelativeSeek() SeekLTFlags {
return s | (1 << seekLTFlagRelativeSeek)
}
// DisableRelativeSeek returns the provided flags with the relative-seek flag
// disabled.
func (s SeekLTFlags) DisableRelativeSeek() SeekLTFlags {
return s &^ (1 << seekLTFlagRelativeSeek)
}
// InternalIteratorStats contains miscellaneous stats produced by
// InternalIterators that are part of the InternalIterator tree. Not every
// field is relevant for an InternalIterator implementation. The field values
// are aggregated as one goes up the InternalIterator tree.
type InternalIteratorStats struct {
// Bytes in the loaded blocks. If the block was compressed, this is the
// compressed bytes. Currently, only the index blocks, data blocks
// containing points, and filter blocks are included.
BlockBytes uint64
// Subset of BlockBytes that were in the block cache.
BlockBytesInCache uint64
// BlockReadDuration accumulates the duration spent fetching blocks
// due to block cache misses.
// TODO(sumeer): this currently excludes the time spent in Reader creation,
// and in reading the rangedel and rangekey blocks. Fix that.
BlockReadDuration time.Duration
// The following can repeatedly count the same points if they are iterated
// over multiple times. Additionally, they may count a point twice when
// switching directions. The latter could be improved if needed.
// Bytes in keys that were iterated over. Currently, only point keys are
// included.
KeyBytes uint64
// Bytes in values that were iterated over. Currently, only point values are
// included. For separated values, this is the size of the handle.
ValueBytes uint64
// The count of points iterated over.
PointCount uint64
// Points that were iterated over that were covered by range tombstones. It
// can be useful for discovering instances of
// https://github.com/cockroachdb/pebble/issues/1070.
PointsCoveredByRangeTombstones uint64
// Stats related to points in value blocks encountered during iteration.
// These are useful to understand outliers, since typical user facing
// iteration should tend to only look at the latest point, and hence have
// the following stats close to 0.
SeparatedPointValue struct {
// Count is a count of points that were in value blocks. This is not a
// subset of PointCount: PointCount is produced by mergingIter and if
// positioned once, and successful in returning a point, will have a
// PointCount of 1, regardless of how many sstables (and memtables etc.)
// in the heap got positioned. The count here includes every sstable
// iterator that got positioned in the heap.
Count uint64
// ValueBytes represent the total byte length of the values (in value
// blocks) of the points corresponding to Count.
ValueBytes uint64
// ValueBytesFetched is the total byte length of the values (in value
// blocks) that were retrieved.
ValueBytesFetched uint64
}
}
// Merge merges the stats in from into the given stats.
func (s *InternalIteratorStats) Merge(from InternalIteratorStats) {
s.BlockBytes += from.BlockBytes
s.BlockBytesInCache += from.BlockBytesInCache
s.BlockReadDuration += from.BlockReadDuration
s.KeyBytes += from.KeyBytes
s.ValueBytes += from.ValueBytes
s.PointCount += from.PointCount
s.PointsCoveredByRangeTombstones += from.PointsCoveredByRangeTombstones
s.SeparatedPointValue.Count += from.SeparatedPointValue.Count
s.SeparatedPointValue.ValueBytes += from.SeparatedPointValue.ValueBytes
s.SeparatedPointValue.ValueBytesFetched += from.SeparatedPointValue.ValueBytesFetched
}

287
vendor/github.com/cockroachdb/pebble/internal/base/lazy_value.go generated vendored Normal file
View File

@@ -0,0 +1,287 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "github.com/cockroachdb/pebble/internal/invariants"
// A value can have user-defined attributes that are a function of the value
// byte slice. For now, we only support "short attributes", which can be
// encoded in 3 bits. We will likely extend this to "long attributes" later
// for values that are even more expensive to access than those in value
// blocks in the same sstable.
//
// When a sstable writer chooses not to store a value together with the key,
// it can call the ShortAttributeExtractor to extract the attribute and store
// it together with the key. This allows for cheap retrieval of
// AttributeAndLen on the read-path, without doing a more expensive retrieval
// of the value. In general, the extraction code may want to also look at the
// key to decide how to treat the value, hence the key* parameters.
//
// Write path performance: The ShortAttributeExtractor func cannot be inlined,
// so we will pay the cost of this function call. However, we will only pay
// this when (a) the value is not being stored together with the key, and (b)
// the key-value pair is being initially written to the DB, or a compaction is
// transitioning the key-value pair from being stored together to being stored
// separately.
// ShortAttribute encodes a user-specified attribute of the value.
type ShortAttribute uint8
// MaxShortAttribute is the maximum value of the short attribute (3 bits).
const MaxShortAttribute = 7
// ShortAttributeExtractor is an extractor that given the value, will return
// the ShortAttribute.
type ShortAttributeExtractor func(
key []byte, keyPrefixLen int, value []byte) (ShortAttribute, error)
// AttributeAndLen represents the pair of value length and the short
// attribute.
type AttributeAndLen struct {
ValueLen int32
ShortAttribute ShortAttribute
}
// LazyValue represents a value that may not already have been extracted.
// Currently, it can represent either an in-place value (stored with the key)
// or a value stored in the value section. However, the interface is general
// enough to support values that are stored in separate files.
//
// LazyValue is used in the InternalIterator interface, such that all
// positioning calls return (*InternalKey, LazyValue). It is also exposed via
// the public Iterator for callers that need to remember a recent but not
// necessarily latest LazyValue, in case they need the actual value in the
// future. An example is a caller that is iterating in reverse and looking for
// the latest MVCC version for a key -- it cannot identify the latest MVCC
// version without stepping to the previous key-value pair e.g.
// storage.pebbleMVCCScanner in CockroachDB.
//
// Performance note: It is important for this struct to not exceed a sizeof 32
// bytes, for optimizing the common case of the in-place value. Prior to
// introducing LazyValue, we were passing around a []byte which is 24 bytes.
// Passing a 40 byte or larger struct causes performance to drop by 75% on
// some benchmarks that do tight iteration loops.
//
// Memory management:
// This is subtle, but important for performance.
//
// A LazyValue returned by an InternalIterator or Iterator is unstable in that
// repositioning the iterator will invalidate the memory inside it. A caller
// wishing to maintain that LazyValue needs to call LazyValue.Clone(). Note
// that this does not fetch the value if it is not in-place. Clone() should
// ideally not be called if LazyValue.Value() has been called, since the
// cloned LazyValue will forget the extracted/fetched value, and calling
// Value() on this clone will cause the value to be extracted again. That is,
// Clone() does not make any promise about the memory stability of the
// underlying value.
//
// A user of an iterator that calls LazyValue.Value() wants as much as
// possible for the returned value []byte to point to iterator owned memory.
//
// 1. [P1] The underlying iterator that owns that memory also needs a promise
// from that user that at any time there is at most one value []byte slice
// that the caller is expecting it to maintain. Otherwise, the underlying
// iterator has to maintain multiple such []byte slices which results in
// more complicated and inefficient code.
//
// 2. [P2] The underlying iterator, in order to make the promise that it is
// maintaining the one value []byte slice, also needs a way to know when
// it is relieved of that promise. One way it is relieved of that promise
// is by being told that it is being repositioned. Typically, the owner of
// the value []byte slice is a sstable iterator, and it will know that it
// is relieved of the promise when it is repositioned. However, consider
// the case where the caller has used LazyValue.Clone() and repositioned
// the iterator (which is actually a tree of iterators). In this case the
// underlying sstable iterator may not even be open. LazyValue.Value()
// will still work (at a higher cost), but since the sstable iterator is
// not open, it does not have a mechanism to know when the retrieved value
// is no longer in use. We refer to this situation as "not satisfying P2".
// To handle this situation, the LazyValue.Value() method accepts a caller
// owned buffer, that the callee will use if needed. The callee explicitly
// tells the caller whether the []byte slice for the value is now owned by
// the caller. This will be true if the callee attempted to use buf and
// either successfully used it or allocated a new []byte slice.
//
// To ground the above in reality, we consider three examples of callers of
// LazyValue.Value():
//
// - Iterator: it calls LazyValue.Value for its own use when merging values.
// When merging during reverse iteration, it may have cloned the LazyValue.
// In this case it calls LazyValue.Value() on the cloned value, merges it,
// and then calls LazyValue.Value() on the current iterator position and
// merges it. So it is honoring P1.
//
// - Iterator on behalf of Iterator clients: The Iterator.Value() method
// needs to call LazyValue.Value(). The client of Iterator is satisfying P1
// because of the inherent Iterator interface constraint, i.e., it is calling
// Iterator.Value() on the current Iterator position. It is possible that
// the Iterator has cloned this LazyValue (for the reverse iteration case),
// which the client is unaware of, so the underlying sstable iterator may
// not be able to satisfy P2. This is ok because Iterator will call
// LazyValue.Value with its (reusable) owned buffer.
//
// - CockroachDB's pebbleMVCCScanner: This will use LazyValues from Iterator
// since during reverse iteration in order to find the highest version that
// satisfies a read it needs to clone the LazyValue, step back the iterator
// and then decide whether it needs the value from the previously cloned
// LazyValue. The pebbleMVCCScanner will satisfy P1. The P2 story is
// similar to the previous case in that it will call LazyValue.Value with
// its (reusable) owned buffer.
//
// Corollary: callers that directly use InternalIterator can know that they
// have done nothing to interfere with promise P2 can pass in a nil buf and be
// sure that it will not trigger an allocation.
//
// Repeated calling of LazyValue.Value:
// This is ok as long as the caller continues to satisfy P1. The previously
// fetched value will be remembered inside LazyValue to avoid fetching again.
// So if the caller's buffer is used the first time the value was fetched, it
// is still in use.
//
// LazyValue fields are visible outside the package for use in
// InternalIterator implementations and in Iterator, but not meant for direct
// use by users of Pebble.
type LazyValue struct {
// ValueOrHandle represents a value, or a handle to be passed to ValueFetcher.
// - Fetcher == nil: ValueOrHandle is a value.
// - Fetcher != nil: ValueOrHandle is a handle and Fetcher.Attribute is
// initialized.
// The ValueOrHandle exposed by InternalIterator or Iterator may not be stable
// if the iterator is stepped. To make it stable, make a copy using Clone.
ValueOrHandle []byte
// Fetcher provides support for fetching an actually lazy value.
Fetcher *LazyFetcher
}
// LazyFetcher supports fetching a lazy value.
//
// Fetcher and Attribute are to be initialized at creation time. The fields
// are arranged to reduce the sizeof this struct.
type LazyFetcher struct {
// Fetcher, given a handle, returns the value.
Fetcher ValueFetcher
err error
value []byte
// Attribute includes the short attribute and value length.
Attribute AttributeAndLen
fetched bool
callerOwned bool
}
// ValueFetcher is an interface for fetching a value.
type ValueFetcher interface {
// Fetch returns the value, given the handle. It is acceptable to call the
// ValueFetcher.Fetch as long as the DB is open. However, one should assume
// there is a fast-path when the iterator tree has not moved off the sstable
// iterator that initially provided this LazyValue. Hence, to utilize this
// fast-path the caller should try to decide whether it needs the value or
// not as soon as possible, with minimal possible stepping of the iterator.
//
// buf will be used if the fetcher cannot satisfy P2 (see earlier comment).
// If the fetcher attempted to use buf *and* len(buf) was insufficient, it
// will allocate a new slice for the value. In either case it will set
// callerOwned to true.
Fetch(
handle []byte, valLen int32, buf []byte) (val []byte, callerOwned bool, err error)
}
// Value returns the underlying value.
func (lv *LazyValue) Value(buf []byte) (val []byte, callerOwned bool, err error) {
if lv.Fetcher == nil {
return lv.ValueOrHandle, false, nil
}
// Do the rest of the work in a separate method to attempt mid-stack
// inlining of Value(). Unfortunately, this still does not inline since the
// cost of 85 exceeds the budget of 80.
//
// TODO(sumeer): Packing the return values into a struct{[]byte error bool}
// causes it to be below the budget. Consider this if we need to recover
// more performance. I suspect that inlining this only matters in
// micro-benchmarks, and in actual use cases in CockroachDB it will not
// matter because there is substantial work done with a fetched value.
return lv.fetchValue(buf)
}
// INVARIANT: lv.Fetcher != nil
func (lv *LazyValue) fetchValue(buf []byte) (val []byte, callerOwned bool, err error) {
f := lv.Fetcher
if !f.fetched {
f.fetched = true
f.value, f.callerOwned, f.err = f.Fetcher.Fetch(
lv.ValueOrHandle, lv.Fetcher.Attribute.ValueLen, buf)
}
return f.value, f.callerOwned, f.err
}
// InPlaceValue returns the value under the assumption that it is in-place.
// This is for Pebble-internal code.
func (lv *LazyValue) InPlaceValue() []byte {
if invariants.Enabled && lv.Fetcher != nil {
panic("value must be in-place")
}
return lv.ValueOrHandle
}
// Len returns the length of the value.
func (lv *LazyValue) Len() int {
if lv.Fetcher == nil {
return len(lv.ValueOrHandle)
}
return int(lv.Fetcher.Attribute.ValueLen)
}
// TryGetShortAttribute returns the ShortAttribute and a bool indicating
// whether the ShortAttribute was populated.
func (lv *LazyValue) TryGetShortAttribute() (ShortAttribute, bool) {
if lv.Fetcher == nil {
return 0, false
}
return lv.Fetcher.Attribute.ShortAttribute, true
}
// Clone creates a stable copy of the LazyValue, by appending bytes to buf.
// The fetcher parameter must be non-nil and may be over-written and used
// inside the returned LazyValue -- this is needed to avoid an allocation.
// Most callers have at most K cloned LazyValues, where K is hard-coded, so
// they can have a pool of exactly K LazyFetcher structs they can reuse in
// these calls. The alternative of allocating LazyFetchers from a sync.Pool is
// not viable since we have no code trigger for returning to the pool
// (LazyValues are simply GC'd).
//
// NB: It is highly preferable that LazyValue.Value() has not been called,
// since the Clone will forget any previously extracted value, and a future
// call to Value will cause it to be fetched again. We do this since we don't
// want to reason about whether or not to clone an already extracted value
// inside the Fetcher (we don't). Property P1 applies here too: if lv1.Value()
// has been called, and then lv2 is created as a clone of lv1, then calling
// lv2.Value() can invalidate any backing memory maintained inside the fetcher
// for lv1 (even though these are the same values). We initially prohibited
// calling LazyValue.Clone() if LazyValue.Value() has been called, but there
// is at least one complex caller (pebbleMVCCScanner inside CockroachDB) where
// it is not easy to prove this invariant.
func (lv *LazyValue) Clone(buf []byte, fetcher *LazyFetcher) (LazyValue, []byte) {
var lvCopy LazyValue
if lv.Fetcher != nil {
*fetcher = LazyFetcher{
Fetcher: lv.Fetcher.Fetcher,
Attribute: lv.Fetcher.Attribute,
// Not copying anything that has been extracted.
}
lvCopy.Fetcher = fetcher
}
vLen := len(lv.ValueOrHandle)
if vLen == 0 {
return lvCopy, buf
}
bufLen := len(buf)
buf = append(buf, lv.ValueOrHandle...)
lvCopy.ValueOrHandle = buf[bufLen : bufLen+vLen]
return lvCopy, buf
}
// MakeInPlaceValue constructs an in-place value.
func MakeInPlaceValue(val []byte) LazyValue {
return LazyValue{ValueOrHandle: val}
}

144
vendor/github.com/cockroachdb/pebble/internal/base/logger.go generated vendored Normal file
View File

@@ -0,0 +1,144 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import (
"bytes"
"context"
"fmt"
"log"
"os"
"runtime"
"sync"
"github.com/cockroachdb/pebble/internal/invariants"
)
// Logger defines an interface for writing log messages.
type Logger interface {
Infof(format string, args ...interface{})
Fatalf(format string, args ...interface{})
}
type defaultLogger struct{}
// DefaultLogger logs to the Go stdlib logs.
var DefaultLogger defaultLogger
var _ Logger = DefaultLogger
// Infof implements the Logger.Infof interface.
func (defaultLogger) Infof(format string, args ...interface{}) {
_ = log.Output(2, fmt.Sprintf(format, args...))
}
// Fatalf implements the Logger.Fatalf interface.
func (defaultLogger) Fatalf(format string, args ...interface{}) {
_ = log.Output(2, fmt.Sprintf(format, args...))
os.Exit(1)
}
// InMemLogger implements Logger using an in-memory buffer (used for testing).
// The buffer can be read via String() and cleared via Reset().
type InMemLogger struct {
mu struct {
sync.Mutex
buf bytes.Buffer
}
}
var _ Logger = (*InMemLogger)(nil)
// Reset clears the internal buffer.
func (b *InMemLogger) Reset() {
b.mu.Lock()
defer b.mu.Unlock()
b.mu.buf.Reset()
}
// String returns the current internal buffer.
func (b *InMemLogger) String() string {
b.mu.Lock()
defer b.mu.Unlock()
return b.mu.buf.String()
}
// Infof is part of the Logger interface.
func (b *InMemLogger) Infof(format string, args ...interface{}) {
s := fmt.Sprintf(format, args...)
b.mu.Lock()
defer b.mu.Unlock()
b.mu.buf.Write([]byte(s))
if n := len(s); n == 0 || s[n-1] != '\n' {
b.mu.buf.Write([]byte("\n"))
}
}
// Fatalf is part of the Logger interface.
func (b *InMemLogger) Fatalf(format string, args ...interface{}) {
b.Infof(format, args...)
runtime.Goexit()
}
// LoggerAndTracer defines an interface for logging and tracing.
type LoggerAndTracer interface {
Logger
// Eventf formats and emits a tracing log, if tracing is enabled in the
// current context.
Eventf(ctx context.Context, format string, args ...interface{})
// IsTracingEnabled returns true if tracing is enabled. It can be used as an
// optimization to avoid calling Eventf (which will be a noop when tracing
// is not enabled) to avoid the overhead of boxing the args.
IsTracingEnabled(ctx context.Context) bool
}
// LoggerWithNoopTracer wraps a logger and does no tracing.
type LoggerWithNoopTracer struct {
Logger
}
var _ LoggerAndTracer = &LoggerWithNoopTracer{}
// Eventf implements LoggerAndTracer.
func (*LoggerWithNoopTracer) Eventf(ctx context.Context, format string, args ...interface{}) {
if invariants.Enabled && ctx == nil {
panic("Eventf context is nil")
}
}
// IsTracingEnabled implements LoggerAndTracer.
func (*LoggerWithNoopTracer) IsTracingEnabled(ctx context.Context) bool {
if invariants.Enabled && ctx == nil {
panic("IsTracingEnabled ctx is nil")
}
return false
}
// NoopLoggerAndTracer does no logging and tracing. Remember that struct{} is
// special cased in Go and does not incur an allocation when it backs the
// interface LoggerAndTracer.
type NoopLoggerAndTracer struct{}
var _ LoggerAndTracer = NoopLoggerAndTracer{}
// Infof implements LoggerAndTracer.
func (l NoopLoggerAndTracer) Infof(format string, args ...interface{}) {}
// Fatalf implements LoggerAndTracer.
func (l NoopLoggerAndTracer) Fatalf(format string, args ...interface{}) {}
// Eventf implements LoggerAndTracer.
func (l NoopLoggerAndTracer) Eventf(ctx context.Context, format string, args ...interface{}) {
if invariants.Enabled && ctx == nil {
panic("Eventf context is nil")
}
}
// IsTracingEnabled implements LoggerAndTracer.
func (l NoopLoggerAndTracer) IsTracingEnabled(ctx context.Context) bool {
if invariants.Enabled && ctx == nil {
panic("IsTracingEnabled ctx is nil")
}
return false
}

133
vendor/github.com/cockroachdb/pebble/internal/base/merger.go generated vendored Normal file
View File

@@ -0,0 +1,133 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "io"
// Merge creates a ValueMerger for the specified key initialized with the value
// of one merge operand.
type Merge func(key, value []byte) (ValueMerger, error)
// ValueMerger receives merge operands one by one. The operand received is either
// newer or older than all operands received so far as indicated by the function
// names, `MergeNewer()` and `MergeOlder()`. Once all operands have been received,
// the client will invoke `Finish()` to obtain the final result. The order of
// a merge is not changed after the first call to `MergeNewer()` or
// `MergeOlder()`, i.e. the same method is used to submit all operands.
//
// The implementation may choose to merge values into the result immediately upon
// receiving each operand, or buffer operands until Finish() is called. For example,
// buffering may be useful to avoid (de)serializing partial merge results.
//
// The merge operation must be associative. That is, for the values A, B, C:
//
// Merge(A).MergeOlder(B).MergeOlder(C) == Merge(C).MergeNewer(B).MergeNewer(A)
//
// Examples of merge operators are integer addition, list append, and string
// concatenation.
type ValueMerger interface {
// MergeNewer adds an operand that is newer than all existing operands.
// The caller retains ownership of value.
//
// If an error is returned the merge is aborted and no other methods must
// be called.
MergeNewer(value []byte) error
// MergeOlder adds an operand that is older than all existing operands.
// The caller retains ownership of value.
//
// If an error is returned the merge is aborted and no other methods must
// be called.
MergeOlder(value []byte) error
// Finish does any final processing of the added operands and returns a
// result. The caller can assume the returned byte slice will not be mutated.
//
// Finish must be the last function called on the ValueMerger. The caller
// must not call any other ValueMerger functions after calling Finish.
//
// If `includesBase` is true, the oldest merge operand was part of the
// merge. This will always be the true during normal iteration, but may be
// false during compaction when only a subset of operands may be
// available. Note that `includesBase` is set to true conservatively: a false
// value means that we could not definitely determine that the base merge
// operand was included.
//
// If a Closer is returned, the returned slice will remain valid until it is
// closed. The caller must arrange for the closer to be eventually closed.
Finish(includesBase bool) ([]byte, io.Closer, error)
}
// DeletableValueMerger is an extension to ValueMerger which allows indicating that the
// result of a merge operation is non-existent. Such non-existent entries will eventually
// be deleted during compaction. Note that during compaction, non-existence of the result
// of a merge means that the merge operands will not result in any record being output.
// This is not the same as transforming the merge operands into a deletion tombstone, as
// older merge operands will still be visible during iteration. Deletion of the merge operands
// in this way is akin to the way a SingleDelete+Set combine into non-existence while leaving
// older records for the same key unaffected.
type DeletableValueMerger interface {
ValueMerger
// DeletableFinish enables a value merger to indicate that the result of a merge operation
// is non-existent. See Finish for a description of includesBase.
DeletableFinish(includesBase bool) (value []byte, delete bool, closer io.Closer, err error)
}
// Merger defines an associative merge operation. The merge operation merges
// two or more values for a single key. A merge operation is requested by
// writing a value using {Batch,DB}.Merge(). The value at that key is merged
// with any existing value. It is valid to Set a value at a key and then Merge
// a new value. Similar to non-merged values, a merged value can be deleted by
// either Delete or DeleteRange.
//
// The merge operation is invoked when a merge value is encountered during a
// read, either during a compaction or during iteration.
type Merger struct {
Merge Merge
// Name is the name of the merger.
//
// Pebble stores the merger name on disk, and opening a database with a
// different merger from the one it was created with will result in an error.
Name string
}
// AppendValueMerger concatenates merge operands in order from oldest to newest.
type AppendValueMerger struct {
buf []byte
}
// MergeNewer appends value to the result.
func (a *AppendValueMerger) MergeNewer(value []byte) error {
a.buf = append(a.buf, value...)
return nil
}
// MergeOlder prepends value to the result, which involves allocating a new buffer.
func (a *AppendValueMerger) MergeOlder(value []byte) error {
buf := make([]byte, len(a.buf)+len(value))
copy(buf, value)
copy(buf[len(value):], a.buf)
a.buf = buf
return nil
}
// Finish returns the buffer that was constructed on-demand in `Merge{OlderNewer}()` calls.
func (a *AppendValueMerger) Finish(includesBase bool) ([]byte, io.Closer, error) {
return a.buf, nil, nil
}
// DefaultMerger is the default implementation of the Merger interface. It
// concatenates the two values to merge.
var DefaultMerger = &Merger{
Merge: func(key, value []byte) (ValueMerger, error) {
res := &AppendValueMerger{}
res.buf = append(res.buf, value...)
return res, nil
},
Name: "pebble.concatenate",
}

98
vendor/github.com/cockroachdb/pebble/internal/base/metrics.go generated vendored Normal file
View File

@@ -0,0 +1,98 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
import "time"
// ThroughputMetric is used to measure the byte throughput of some component
// that performs work in a single-threaded manner. The throughput can be
// approximated by Bytes/(WorkDuration+IdleTime). The idle time is represented
// separately, so that the user of this metric could approximate the peak
// throughput as Bytes/WorkTime. The metric is designed to be cumulative (see
// Merge).
type ThroughputMetric struct {
// Bytes is the processes bytes by the component.
Bytes int64
// WorkDuration is the duration that the component spent doing work.
WorkDuration time.Duration
// IdleDuration is the duration that the component was idling, waiting for
// work.
IdleDuration time.Duration
}
// Merge accumulates the information from another throughput metric.
func (tm *ThroughputMetric) Merge(x ThroughputMetric) {
tm.Bytes += x.Bytes
tm.WorkDuration += x.WorkDuration
tm.IdleDuration += x.IdleDuration
}
// Subtract subtracts the information from another ThroughputMetric
func (tm *ThroughputMetric) Subtract(x ThroughputMetric) {
tm.Bytes -= x.Bytes
tm.WorkDuration -= x.WorkDuration
tm.IdleDuration -= x.IdleDuration
}
// PeakRate returns the approximate peak rate if there was no idling.
func (tm *ThroughputMetric) PeakRate() int64 {
if tm.Bytes == 0 {
return 0
}
return int64((float64(tm.Bytes) / float64(tm.WorkDuration)) * float64(time.Second))
}
// Rate returns the observed rate.
func (tm *ThroughputMetric) Rate() int64 {
if tm.Bytes == 0 {
return 0
}
return int64((float64(tm.Bytes) / float64(tm.WorkDuration+tm.IdleDuration)) *
float64(time.Second))
}
// Utilization returns a percent [0, 1.0] indicating the percent of time
// work was performed.
func (tm *ThroughputMetric) Utilization() float64 {
if tm.WorkDuration == 0 {
return 0
}
return float64(tm.WorkDuration) / float64(tm.WorkDuration+tm.IdleDuration)
}
// GaugeSampleMetric is used to measure a gauge value (e.g. queue length) by
// accumulating samples of that gauge.
type GaugeSampleMetric struct {
// The sum of all the samples.
sampleSum int64
// The number of samples.
count int64
}
// AddSample adds the given sample.
func (gsm *GaugeSampleMetric) AddSample(sample int64) {
gsm.sampleSum += sample
gsm.count++
}
// Merge accumulates the information from another gauge metric.
func (gsm *GaugeSampleMetric) Merge(x GaugeSampleMetric) {
gsm.sampleSum += x.sampleSum
gsm.count += x.count
}
// Subtract subtracts the information from another gauge metric.
func (gsm *GaugeSampleMetric) Subtract(x GaugeSampleMetric) {
gsm.sampleSum -= x.sampleSum
gsm.count -= x.count
}
// Mean returns the mean value.
func (gsm *GaugeSampleMetric) Mean() float64 {
if gsm.count == 0 {
return 0
}
return float64(gsm.sampleSum) / float64(gsm.count)
}

76
vendor/github.com/cockroachdb/pebble/internal/base/options.go generated vendored Normal file
View File

@@ -0,0 +1,76 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package base
// SSTable block defaults.
const (
DefaultBlockRestartInterval = 16
DefaultBlockSize = 4096
DefaultBlockSizeThreshold = 90
)
// FilterType is the level at which to apply a filter: block or table.
type FilterType int
// The available filter types.
const (
TableFilter FilterType = iota
)
func (t FilterType) String() string {
switch t {
case TableFilter:
return "table"
}
return "unknown"
}
// FilterWriter provides an interface for creating filter blocks. See
// FilterPolicy for more details about filters.
type FilterWriter interface {
// AddKey adds a key to the current filter block.
AddKey(key []byte)
// Finish appends to dst an encoded filter tha holds the current set of
// keys. The writer state is reset after the call to Finish allowing the
// writer to be reused for the creation of additional filters.
Finish(dst []byte) []byte
}
// FilterPolicy is an algorithm for probabilistically encoding a set of keys.
// The canonical implementation is a Bloom filter.
//
// Every FilterPolicy has a name. This names the algorithm itself, not any one
// particular instance. Aspects specific to a particular instance, such as the
// set of keys or any other parameters, will be encoded in the []byte filter
// returned by NewWriter.
//
// The name may be written to files on disk, along with the filter data. To use
// these filters, the FilterPolicy name at the time of writing must equal the
// name at the time of reading. If they do not match, the filters will be
// ignored, which will not affect correctness but may affect performance.
type FilterPolicy interface {
// Name names the filter policy.
Name() string
// MayContain returns whether the encoded filter may contain given key.
// False positives are possible, where it returns true for keys not in the
// original set.
MayContain(ftype FilterType, filter, key []byte) bool
// NewWriter creates a new FilterWriter.
NewWriter(ftype FilterType) FilterWriter
}
// BlockPropertyFilter is used in an Iterator to filter sstables and blocks
// within the sstable. It should not maintain any per-sstable state, and must
// be thread-safe.
type BlockPropertyFilter interface {
// Name returns the name of the block property collector.
Name() string
// Intersects returns true if the set represented by prop intersects with
// the set in the filter.
Intersects(prop []byte) (bool, error)
}

56
vendor/github.com/cockroachdb/pebble/internal/batchskl/README.md generated vendored Normal file
View File

@@ -0,0 +1,56 @@
# batchskl
Fast, non-concurrent skiplist implementation in Go that supports
forward and backward iteration.
## Limitations
* The interface is tailored for use in indexing pebble batches. Keys
and values are stored outside of the skiplist making the skiplist
awkward for general purpose use.
* Deletion is not supported. Instead, higher-level code is expected to
add deletion tombstones and needs to process those tombstones
appropriately.
## Pedigree
This code is based on Andy Kimball's arenaskl code.
The arenaskl code is based on the skiplist found in Badger, a Go-based
KV store:
https://github.com/dgraph-io/badger/tree/master/skl
The skiplist in Badger is itself based on a C++ skiplist built for
Facebook's RocksDB:
https://github.com/facebook/rocksdb/tree/master/memtable
## Benchmarks
The benchmarks consist of a mix of reads and writes executed in parallel. The
fraction of reads is indicated in the run name: "frac_X" indicates a run where
X percent of the operations are reads.
```
name time/op
ReadWrite/frac_0 1.03µs ± 2%
ReadWrite/frac_10 1.32µs ± 1%
ReadWrite/frac_20 1.26µs ± 1%
ReadWrite/frac_30 1.18µs ± 1%
ReadWrite/frac_40 1.09µs ± 1%
ReadWrite/frac_50 987ns ± 2%
ReadWrite/frac_60 1.07µs ± 1%
ReadWrite/frac_70 909ns ± 1%
ReadWrite/frac_80 693ns ± 2%
ReadWrite/frac_90 599ns ± 2%
ReadWrite/frac_100 45.3ns ± 3%
```
Forward and backward iteration are also fast:
```
name time/op
IterNext 4.49ns ± 3%
IterPrev 4.48ns ± 3%
```

223
vendor/github.com/cockroachdb/pebble/internal/batchskl/iterator.go generated vendored Normal file
View File

@@ -0,0 +1,223 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package batchskl
import "github.com/cockroachdb/pebble/internal/base"
type splice struct {
prev uint32
next uint32
}
// Iterator is an iterator over the skiplist object. Use Skiplist.NewIter
// to construct an iterator. The current state of the iterator can be cloned
// by simply value copying the struct.
type Iterator struct {
list *Skiplist
nd uint32
key base.InternalKey
lower []byte
upper []byte
}
// Close resets the iterator.
func (it *Iterator) Close() error {
it.list = nil
it.nd = 0
return nil
}
// SeekGE moves the iterator to the first entry whose key is greater than or
// equal to the given key. Returns true if the iterator is pointing at a valid
// entry and false otherwise. Note that SeekGE only checks the upper bound. It
// is up to the caller to ensure that key is greater than or equal to the lower
// bound.
func (it *Iterator) SeekGE(key []byte, flags base.SeekGEFlags) *base.InternalKey {
if flags.TrySeekUsingNext() {
if it.nd == it.list.tail {
// Iterator is done.
return nil
}
less := it.list.cmp(it.key.UserKey, key) < 0
// Arbitrary constant. By measuring the seek cost as a function of the
// number of elements in the skip list, and fitting to a model, we
// could adjust the number of nexts based on the current size of the
// skip list.
const numNexts = 5
for i := 0; less && i < numNexts; i++ {
k := it.Next()
if k == nil {
// Iterator is done.
return nil
}
less = it.list.cmp(k.UserKey, key) < 0
}
if !less {
return &it.key
}
}
_, it.nd = it.seekForBaseSplice(key, it.list.abbreviatedKey(key))
if it.nd == it.list.tail {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.upper != nil && it.list.cmp(it.upper, nodeKey.UserKey) <= 0 {
it.nd = it.list.tail
return nil
}
it.key = nodeKey
return &it.key
}
// SeekLT moves the iterator to the last entry whose key is less the given
// key. Returns true if the iterator is pointing at a valid entry and false
// otherwise. Note that SeekLT only checks the lower bound. It is up to the
// caller to ensure that key is less than the upper bound.
func (it *Iterator) SeekLT(key []byte) *base.InternalKey {
it.nd, _ = it.seekForBaseSplice(key, it.list.abbreviatedKey(key))
if it.nd == it.list.head {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.lower != nil && it.list.cmp(it.lower, nodeKey.UserKey) > 0 {
it.nd = it.list.head
return nil
}
it.key = nodeKey
return &it.key
}
// First seeks position at the first entry in list. Final state of iterator is
// Valid() iff list is not empty. Note that First only checks the upper
// bound. It is up to the caller to ensure that key is greater than or equal to
// the lower bound (e.g. via a call to SeekGE(lower)).
func (it *Iterator) First() *base.InternalKey {
it.nd = it.list.getNext(it.list.head, 0)
if it.nd == it.list.tail {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.upper != nil && it.list.cmp(it.upper, nodeKey.UserKey) <= 0 {
it.nd = it.list.tail
return nil
}
it.key = nodeKey
return &it.key
}
// Last seeks position at the last entry in list. Final state of iterator is
// Valid() iff list is not empty. Note that Last only checks the lower
// bound. It is up to the caller to ensure that key is less than the upper
// bound (e.g. via a call to SeekLT(upper)).
func (it *Iterator) Last() *base.InternalKey {
it.nd = it.list.getPrev(it.list.tail, 0)
if it.nd == it.list.head {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.lower != nil && it.list.cmp(it.lower, nodeKey.UserKey) > 0 {
it.nd = it.list.head
return nil
}
it.key = nodeKey
return &it.key
}
// Next advances to the next position. If there are no following nodes, then
// Valid() will be false after this call.
func (it *Iterator) Next() *base.InternalKey {
it.nd = it.list.getNext(it.nd, 0)
if it.nd == it.list.tail {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.upper != nil && it.list.cmp(it.upper, nodeKey.UserKey) <= 0 {
it.nd = it.list.tail
return nil
}
it.key = nodeKey
return &it.key
}
// Prev moves to the previous position. If there are no previous nodes, then
// Valid() will be false after this call.
func (it *Iterator) Prev() *base.InternalKey {
it.nd = it.list.getPrev(it.nd, 0)
if it.nd == it.list.head {
return nil
}
nodeKey := it.list.getKey(it.nd)
if it.lower != nil && it.list.cmp(it.lower, nodeKey.UserKey) > 0 {
it.nd = it.list.head
return nil
}
it.key = nodeKey
return &it.key
}
// Key returns the key at the current position.
func (it *Iterator) Key() *base.InternalKey {
return &it.key
}
// KeyInfo returns the offset of the start of the record, the start of the key,
// and the end of the key.
func (it *Iterator) KeyInfo() (offset, keyStart, keyEnd uint32) {
n := it.list.node(it.nd)
return n.offset, n.keyStart, n.keyEnd
}
// Head true iff the iterator is positioned at the sentinel head node.
func (it *Iterator) Head() bool {
return it.nd == it.list.head
}
// Tail true iff the iterator is positioned at the sentinel tail node.
func (it *Iterator) Tail() bool {
return it.nd == it.list.tail
}
// Valid returns nil iff the iterator is positioned at a valid node.
func (it *Iterator) Valid() bool {
return it.list != nil && it.nd != it.list.head && it.nd != it.list.tail
}
func (it *Iterator) String() string {
return "batch"
}
// SetBounds sets the lower and upper bounds for the iterator. Note that the
// result of Next and Prev will be undefined until the iterator has been
// repositioned with SeekGE, SeekLT, First, or Last.
func (it *Iterator) SetBounds(lower, upper []byte) {
it.lower = lower
it.upper = upper
}
func (it *Iterator) seekForBaseSplice(key []byte, abbreviatedKey uint64) (prev, next uint32) {
prev = it.list.head
for level := it.list.height - 1; ; level-- {
prev, next = it.list.findSpliceForLevel(key, abbreviatedKey, level, prev)
if level == 0 {
break
}
}
return
}

442
vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go generated vendored Normal file
View File

@@ -0,0 +1,442 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
* Modifications copyright (C) 2017 Andy Kimball and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
Adapted from RocksDB inline skiplist.
Key differences:
- No optimization for sequential inserts (no "prev").
- No custom comparator.
- Support overwrites. This requires care when we see the same key when inserting.
For RocksDB or LevelDB, overwrites are implemented as a newer sequence number in the key, so
there is no need for values. We don't intend to support versioning. In-place updates of values
would be more efficient.
- We discard all non-concurrent code.
- We do not support Splices. This simplifies the code a lot.
- No AllocateNode or other pointer arithmetic.
- We combine the findLessThan, findGreaterOrEqual, etc into one function.
*/
/*
Further adapted from Badger: https://github.com/dgraph-io/badger.
Key differences:
- Support for previous pointers - doubly linked lists. Note that it's up to higher
level code to deal with the intermediate state that occurs during insertion,
where node A is linked to node B, but node B is not yet linked back to node A.
- Iterator includes mutator functions.
*/
/*
Further adapted from arenaskl: https://github.com/andy-kimball/arenaskl
Key differences:
- Removed support for deletion.
- Removed support for concurrency.
- External storage of keys.
- Node storage grows to an arbitrary size.
*/
package batchskl // import "github.com/cockroachdb/pebble/internal/batchskl"
import (
"bytes"
"encoding/binary"
"fmt"
"math"
"time"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/constants"
"golang.org/x/exp/rand"
)
const (
maxHeight = 20
maxNodeSize = uint64(unsafe.Sizeof(node{}))
linksSize = uint64(unsafe.Sizeof(links{}))
maxNodesSize = constants.MaxUint32OrInt
)
var (
// ErrExists indicates that a duplicate record was inserted. This should never
// happen for normal usage of batchskl as every key should have a unique
// sequence number.
ErrExists = errors.New("record with this key already exists")
// ErrTooManyRecords is a sentinel error returned when the size of the raw
// nodes slice exceeds the maximum allowed size (currently 1 << 32 - 1). This
// corresponds to ~117 M skiplist entries.
ErrTooManyRecords = errors.New("too many records")
)
type links struct {
next uint32
prev uint32
}
type node struct {
// The offset of the start of the record in the storage.
offset uint32
// The offset of the start and end of the key in storage.
keyStart uint32
keyEnd uint32
// A fixed 8-byte abbreviation of the key, used to avoid retrieval of the key
// during seek operations. The key retrieval can be expensive purely due to
// cache misses while the abbreviatedKey stored here will be in the same
// cache line as the key and the links making accessing and comparing against
// it almost free.
abbreviatedKey uint64
// Most nodes do not need to use the full height of the link tower, since the
// probability of each successive level decreases exponentially. Because
// these elements are never accessed, they do not need to be allocated.
// Therefore, when a node is allocated, its memory footprint is deliberately
// truncated to not include unneeded link elements.
links [maxHeight]links
}
// Skiplist is a fast, non-cocnurrent skiplist implementation that supports
// forward and backward iteration. See arenaskl.Skiplist for a concurrent
// skiplist. Keys and values are stored externally from the skiplist via the
// Storage interface. Deletion is not supported. Instead, higher-level code is
// expected to perform deletion via tombstones and needs to process those
// tombstones appropriately during retrieval operations.
type Skiplist struct {
storage *[]byte
cmp base.Compare
abbreviatedKey base.AbbreviatedKey
nodes []byte
head uint32
tail uint32
height uint32 // Current height: 1 <= height <= maxHeight
rand rand.PCGSource
}
var (
probabilities [maxHeight]uint32
)
func init() {
const pValue = 1 / math.E
// Precompute the skiplist probabilities so that only a single random number
// needs to be generated and so that the optimal pvalue can be used (inverse
// of Euler's number).
p := float64(1.0)
for i := 0; i < maxHeight; i++ {
probabilities[i] = uint32(float64(math.MaxUint32) * p)
p *= pValue
}
}
// NewSkiplist constructs and initializes a new, empty skiplist.
func NewSkiplist(storage *[]byte, cmp base.Compare, abbreviatedKey base.AbbreviatedKey) *Skiplist {
s := &Skiplist{}
s.Init(storage, cmp, abbreviatedKey)
return s
}
// Reset the fields in the skiplist for reuse.
func (s *Skiplist) Reset() {
*s = Skiplist{
nodes: s.nodes[:0],
height: 1,
}
const batchMaxRetainedSize = 1 << 20 // 1 MB
if cap(s.nodes) > batchMaxRetainedSize {
s.nodes = nil
}
}
// Init the skiplist to empty and re-initialize.
func (s *Skiplist) Init(storage *[]byte, cmp base.Compare, abbreviatedKey base.AbbreviatedKey) {
*s = Skiplist{
storage: storage,
cmp: cmp,
abbreviatedKey: abbreviatedKey,
nodes: s.nodes[:0],
height: 1,
}
s.rand.Seed(uint64(time.Now().UnixNano()))
const initBufSize = 256
if cap(s.nodes) < initBufSize {
s.nodes = make([]byte, 0, initBufSize)
}
// Allocate head and tail nodes. While allocating a new node can fail, in the
// context of initializing the skiplist we consider it unrecoverable.
var err error
s.head, err = s.newNode(maxHeight, 0, 0, 0, 0)
if err != nil {
panic(err)
}
s.tail, err = s.newNode(maxHeight, 0, 0, 0, 0)
if err != nil {
panic(err)
}
// Link all head/tail levels together.
headNode := s.node(s.head)
tailNode := s.node(s.tail)
for i := uint32(0); i < maxHeight; i++ {
headNode.links[i].next = s.tail
tailNode.links[i].prev = s.head
}
}
// Add adds a new key to the skiplist if it does not yet exist. If the record
// already exists, then Add returns ErrRecordExists.
func (s *Skiplist) Add(keyOffset uint32) error {
data := (*s.storage)[keyOffset+1:]
v, n := binary.Uvarint(data)
if n <= 0 {
return errors.Errorf("corrupted batch entry: %d", errors.Safe(keyOffset))
}
data = data[n:]
if v > uint64(len(data)) {
return errors.Errorf("corrupted batch entry: %d", errors.Safe(keyOffset))
}
keyStart := 1 + keyOffset + uint32(n)
keyEnd := keyStart + uint32(v)
key := data[:v]
abbreviatedKey := s.abbreviatedKey(key)
// spl holds the list of next and previous links for each level in the
// skiplist indicating where the new node will be inserted.
var spl [maxHeight]splice
// Fast-path for in-order insertion of keys: compare the new key against the
// last key.
prev := s.getPrev(s.tail, 0)
if prevNode := s.node(prev); prev == s.head ||
abbreviatedKey > prevNode.abbreviatedKey ||
(abbreviatedKey == prevNode.abbreviatedKey &&
s.cmp(key, (*s.storage)[prevNode.keyStart:prevNode.keyEnd]) > 0) {
for level := uint32(0); level < s.height; level++ {
spl[level].prev = s.getPrev(s.tail, level)
spl[level].next = s.tail
}
} else {
s.findSplice(key, abbreviatedKey, &spl)
}
height := s.randomHeight()
// Increase s.height as necessary.
for ; s.height < height; s.height++ {
spl[s.height].next = s.tail
spl[s.height].prev = s.head
}
// We always insert from the base level and up. After you add a node in base
// level, we cannot create a node in the level above because it would have
// discovered the node in the base level.
nd, err := s.newNode(height, keyOffset, keyStart, keyEnd, abbreviatedKey)
if err != nil {
return err
}
newNode := s.node(nd)
for level := uint32(0); level < height; level++ {
next := spl[level].next
prev := spl[level].prev
newNode.links[level].next = next
newNode.links[level].prev = prev
s.node(next).links[level].prev = nd
s.node(prev).links[level].next = nd
}
return nil
}
// NewIter returns a new Iterator object. The lower and upper bound parameters
// control the range of keys the iterator will return. Specifying for nil for
// lower or upper bound disables the check for that boundary. Note that lower
// bound is not checked on {SeekGE,First} and upper bound is not check on
// {SeekLT,Last}. The user is expected to perform that check. Note that it is
// safe for an iterator to be copied by value.
func (s *Skiplist) NewIter(lower, upper []byte) Iterator {
return Iterator{list: s, lower: lower, upper: upper}
}
func (s *Skiplist) newNode(
height,
offset, keyStart, keyEnd uint32, abbreviatedKey uint64,
) (uint32, error) {
if height < 1 || height > maxHeight {
panic("height cannot be less than one or greater than the max height")
}
unusedSize := uint64(maxHeight-int(height)) * linksSize
nodeOffset, err := s.alloc(uint32(maxNodeSize - unusedSize))
if err != nil {
return 0, err
}
nd := s.node(nodeOffset)
nd.offset = offset
nd.keyStart = keyStart
nd.keyEnd = keyEnd
nd.abbreviatedKey = abbreviatedKey
return nodeOffset, nil
}
func (s *Skiplist) alloc(size uint32) (uint32, error) {
offset := uint64(len(s.nodes))
// We only have a need for memory up to offset + size, but we never want
// to allocate a node whose tail points into unallocated memory.
minAllocSize := offset + maxNodeSize
if uint64(cap(s.nodes)) < minAllocSize {
allocSize := uint64(cap(s.nodes)) * 2
if allocSize < minAllocSize {
allocSize = minAllocSize
}
// Cap the allocation at the max allowed size to avoid wasted capacity.
if allocSize > maxNodesSize {
// The new record may still not fit within the allocation, in which case
// we return early with an error. This avoids the panic below when we
// resize the slice. It also avoids the allocation and copy.
if uint64(offset)+uint64(size) > maxNodesSize {
return 0, errors.Wrapf(ErrTooManyRecords,
"alloc of new record (size=%d) would overflow uint32 (current size=%d)",
uint64(offset)+uint64(size), offset,
)
}
allocSize = maxNodesSize
}
tmp := make([]byte, len(s.nodes), allocSize)
copy(tmp, s.nodes)
s.nodes = tmp
}
newSize := uint32(offset) + size
s.nodes = s.nodes[:newSize]
return uint32(offset), nil
}
func (s *Skiplist) node(offset uint32) *node {
return (*node)(unsafe.Pointer(&s.nodes[offset]))
}
func (s *Skiplist) randomHeight() uint32 {
rnd := uint32(s.rand.Uint64())
h := uint32(1)
for h < maxHeight && rnd <= probabilities[h] {
h++
}
return h
}
func (s *Skiplist) findSplice(key []byte, abbreviatedKey uint64, spl *[maxHeight]splice) {
prev := s.head
for level := s.height - 1; ; level-- {
// The code in this loop is the same as findSpliceForLevel(). For some
// reason, calling findSpliceForLevel() here is much much slower than the
// inlined code below. The excess time is also caught up in the final
// return statement which makes little sense. Revisit when in go1.14 or
// later if inlining improves.
next := s.getNext(prev, level)
for next != s.tail {
// Assume prev.key < key.
nextNode := s.node(next)
nextAbbreviatedKey := nextNode.abbreviatedKey
if abbreviatedKey < nextAbbreviatedKey {
// We are done for this level, since prev.key < key < next.key.
break
}
if abbreviatedKey == nextAbbreviatedKey {
if s.cmp(key, (*s.storage)[nextNode.keyStart:nextNode.keyEnd]) <= 0 {
// We are done for this level, since prev.key < key <= next.key.
break
}
}
// Keep moving right on this level.
prev = next
next = nextNode.links[level].next
}
spl[level].prev = prev
spl[level].next = next
if level == 0 {
break
}
}
}
func (s *Skiplist) findSpliceForLevel(
key []byte, abbreviatedKey uint64, level, start uint32,
) (prev, next uint32) {
prev = start
next = s.getNext(prev, level)
for next != s.tail {
// Assume prev.key < key.
nextNode := s.node(next)
nextAbbreviatedKey := nextNode.abbreviatedKey
if abbreviatedKey < nextAbbreviatedKey {
// We are done for this level, since prev.key < key < next.key.
break
}
if abbreviatedKey == nextAbbreviatedKey {
if s.cmp(key, (*s.storage)[nextNode.keyStart:nextNode.keyEnd]) <= 0 {
// We are done for this level, since prev.key < key < next.key.
break
}
}
// Keep moving right on this level.
prev = next
next = nextNode.links[level].next
}
return
}
func (s *Skiplist) getKey(nd uint32) base.InternalKey {
n := s.node(nd)
kind := base.InternalKeyKind((*s.storage)[n.offset])
key := (*s.storage)[n.keyStart:n.keyEnd]
return base.MakeInternalKey(key, uint64(n.offset)|base.InternalKeySeqNumBatch, kind)
}
func (s *Skiplist) getNext(nd, h uint32) uint32 {
return s.node(nd).links[h].next
}
func (s *Skiplist) getPrev(nd, h uint32) uint32 {
return s.node(nd).links[h].prev
}
func (s *Skiplist) debug() string {
var buf bytes.Buffer
for level := uint32(0); level < s.height; level++ {
var count int
for nd := s.head; nd != s.tail; nd = s.getNext(nd, level) {
count++
}
fmt.Fprintf(&buf, "%d: %d\n", level, count)
}
return buf.String()
}
// Silence unused warning.
var _ = (*Skiplist).debug

69
vendor/github.com/cockroachdb/pebble/internal/bytealloc/bytealloc.go generated vendored Normal file
View File

@@ -0,0 +1,69 @@
// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package bytealloc
import "github.com/cockroachdb/pebble/internal/rawalloc"
// An A provides chunk allocation of []byte, amortizing the overhead of each
// allocation. Because the underlying storage for the slices is shared, they
// should share a similar lifetime in order to avoid pinning large amounts of
// memory unnecessarily. The allocator itself is a []byte where cap() indicates
// the total amount of memory and len() is the amount already allocated. The
// size of the buffer to allocate from is grown exponentially when it runs out
// of room up to a maximum size (chunkAllocMaxSize).
type A []byte
const chunkAllocMinSize = 512
const chunkAllocMaxSize = 512 << 10 // 512 KB
func (a A) reserve(n int) A {
allocSize := cap(a) * 2
if allocSize < chunkAllocMinSize {
allocSize = chunkAllocMinSize
} else if allocSize > chunkAllocMaxSize {
allocSize = chunkAllocMaxSize
}
if allocSize < n {
allocSize = n
}
return rawalloc.New(0, allocSize)
}
// Alloc allocates a new chunk of memory with the specified length.
func (a A) Alloc(n int) (A, []byte) {
if cap(a)-len(a) < n {
a = a.reserve(n)
}
p := len(a)
r := a[p : p+n : p+n]
a = a[:p+n]
return a, r
}
// Copy allocates a new chunk of memory, initializing it from src.
func (a A) Copy(src []byte) (A, []byte) {
var alloc []byte
a, alloc = a.Alloc(len(src))
copy(alloc, src)
return a, alloc
}
// Reset returns the current chunk, resetting allocated memory back to none.
// Future allocations will use memory previously allocated by previous calls to
// Alloc or Copy, so the caller must know know that none of the previously
// allocated byte slices are still in use.
func (a A) Reset() A {
return a[:0]
}

21
vendor/github.com/cockroachdb/pebble/internal/cache/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,21 @@
The MIT License
Copyright (c) 2018 Damian Gryski
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

10
vendor/github.com/cockroachdb/pebble/internal/cache/cgo_disabled.go generated vendored Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !cgo
// +build !cgo
package cache
const cgoEnabled = false

10
vendor/github.com/cockroachdb/pebble/internal/cache/cgo_enabled.go generated vendored Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build cgo
// +build cgo
package cache
const cgoEnabled = true

909
vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go generated vendored Normal file
View File

@@ -0,0 +1,909 @@
// Copyright 2018. All rights reserved. Use of this source code is governed by
// an MIT-style license that can be found in the LICENSE file.
// Package cache implements the CLOCK-Pro caching algorithm.
//
// CLOCK-Pro is a patent-free alternative to the Adaptive Replacement Cache,
// https://en.wikipedia.org/wiki/Adaptive_replacement_cache.
// It is an approximation of LIRS ( https://en.wikipedia.org/wiki/LIRS_caching_algorithm ),
// much like the CLOCK page replacement algorithm is an approximation of LRU.
//
// This implementation is based on the python code from https://bitbucket.org/SamiLehtinen/pyclockpro .
//
// Slides describing the algorithm: http://fr.slideshare.net/huliang64/clockpro
//
// The original paper: http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html
//
// It is MIT licensed, like the original.
package cache // import "github.com/cockroachdb/pebble/internal/cache"
import (
"fmt"
"os"
"runtime"
"runtime/debug"
"strings"
"sync"
"sync/atomic"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
type fileKey struct {
// id is the namespace for fileNums.
id uint64
fileNum base.DiskFileNum
}
type key struct {
fileKey
offset uint64
}
// file returns the "file key" for the receiver. This is the key used for the
// shard.files map.
func (k key) file() key {
k.offset = 0
return k
}
func (k key) String() string {
return fmt.Sprintf("%d/%d/%d", k.id, k.fileNum, k.offset)
}
// Handle provides a strong reference to a value in the cache. The reference
// does not pin the value in the cache, but it does prevent the underlying byte
// slice from being reused.
type Handle struct {
value *Value
}
// Get returns the value stored in handle.
func (h Handle) Get() []byte {
if h.value != nil {
// NB: We don't increment shard.hits in this code path because we only want
// to record a hit when the handle is retrieved from the cache.
return h.value.buf
}
return nil
}
// Release releases the reference to the cache entry.
func (h Handle) Release() {
h.value.release()
}
type shard struct {
hits atomic.Int64
misses atomic.Int64
mu sync.RWMutex
reservedSize int64
maxSize int64
coldTarget int64
blocks robinHoodMap // fileNum+offset -> block
files robinHoodMap // fileNum -> list of blocks
// The blocks and files maps store values in manually managed memory that is
// invisible to the Go GC. This is fine for Value and entry objects that are
// stored in manually managed memory, but when the "invariants" build tag is
// set, all Value and entry objects are Go allocated and the entries map will
// contain a reference to every entry.
entries map[*entry]struct{}
handHot *entry
handCold *entry
handTest *entry
sizeHot int64
sizeCold int64
sizeTest int64
// The count fields are used exclusively for asserting expectations.
// We've seen infinite looping (cockroachdb/cockroach#70154) that
// could be explained by a corrupted sizeCold. Through asserting on
// these fields, we hope to gain more insight from any future
// reproductions.
countHot int64
countCold int64
countTest int64
}
func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
c.mu.RLock()
var value *Value
if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil {
value = e.acquireValue()
if value != nil {
e.referenced.Store(true)
}
}
c.mu.RUnlock()
if value == nil {
c.misses.Add(1)
return Handle{}
}
c.hits.Add(1)
return Handle{value: value}
}
func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
if n := value.refs(); n != 1 {
panic(fmt.Sprintf("pebble: Value has already been added to the cache: refs=%d", n))
}
c.mu.Lock()
defer c.mu.Unlock()
k := key{fileKey{id, fileNum}, offset}
e := c.blocks.Get(k)
switch {
case e == nil:
// no cache entry? add it
e = newEntry(c, k, int64(len(value.buf)))
e.setValue(value)
if c.metaAdd(k, e) {
value.ref.trace("add-cold")
c.sizeCold += e.size
c.countCold++
} else {
value.ref.trace("skip-cold")
e.free()
e = nil
}
case e.peekValue() != nil:
// cache entry was a hot or cold page
e.setValue(value)
e.referenced.Store(true)
delta := int64(len(value.buf)) - e.size
e.size = int64(len(value.buf))
if e.ptype == etHot {
value.ref.trace("add-hot")
c.sizeHot += delta
} else {
value.ref.trace("add-cold")
c.sizeCold += delta
}
c.evict()
default:
// cache entry was a test page
c.sizeTest -= e.size
c.countTest--
c.metaDel(e).release()
c.metaCheck(e)
e.size = int64(len(value.buf))
c.coldTarget += e.size
if c.coldTarget > c.targetSize() {
c.coldTarget = c.targetSize()
}
e.referenced.Store(false)
e.setValue(value)
e.ptype = etHot
if c.metaAdd(k, e) {
value.ref.trace("add-hot")
c.sizeHot += e.size
c.countHot++
} else {
value.ref.trace("skip-hot")
e.free()
e = nil
}
}
c.checkConsistency()
// Values are initialized with a reference count of 1. That reference count
// is being transferred to the returned Handle.
return Handle{value: value}
}
func (c *shard) checkConsistency() {
// See the comment above the count{Hot,Cold,Test} fields.
switch {
case c.sizeHot < 0 || c.sizeCold < 0 || c.sizeTest < 0 || c.countHot < 0 || c.countCold < 0 || c.countTest < 0:
panic(fmt.Sprintf("pebble: unexpected negative: %d (%d bytes) hot, %d (%d bytes) cold, %d (%d bytes) test",
c.countHot, c.sizeHot, c.countCold, c.sizeCold, c.countTest, c.sizeTest))
case c.sizeHot > 0 && c.countHot == 0:
panic(fmt.Sprintf("pebble: mismatch %d hot size, %d hot count", c.sizeHot, c.countHot))
case c.sizeCold > 0 && c.countCold == 0:
panic(fmt.Sprintf("pebble: mismatch %d cold size, %d cold count", c.sizeCold, c.countCold))
case c.sizeTest > 0 && c.countTest == 0:
panic(fmt.Sprintf("pebble: mismatch %d test size, %d test count", c.sizeTest, c.countTest))
}
}
// Delete deletes the cached value for the specified file and offset.
func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
// The common case is there is nothing to delete, so do a quick check with
// shared lock.
k := key{fileKey{id, fileNum}, offset}
c.mu.RLock()
exists := c.blocks.Get(k) != nil
c.mu.RUnlock()
if !exists {
return
}
var deletedValue *Value
func() {
c.mu.Lock()
defer c.mu.Unlock()
e := c.blocks.Get(k)
if e == nil {
return
}
deletedValue = c.metaEvict(e)
c.checkConsistency()
}()
// Now that the mutex has been dropped, release the reference which will
// potentially free the memory associated with the previous cached value.
deletedValue.release()
}
// EvictFile evicts all of the cache values for the specified file.
func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) {
fkey := key{fileKey{id, fileNum}, 0}
for c.evictFileRun(fkey) {
// Sched switch to give another goroutine an opportunity to acquire the
// shard mutex.
runtime.Gosched()
}
}
func (c *shard) evictFileRun(fkey key) (moreRemaining bool) {
// If most of the file's blocks are held in the block cache, evicting all
// the blocks may take a while. We don't want to block the entire cache
// shard, forcing concurrent readers to wait until we're finished. We drop
// the mutex every [blocksPerMutexAcquisition] blocks to give other
// goroutines an opportunity to make progress.
const blocksPerMutexAcquisition = 5
c.mu.Lock()
// Releasing a value may result in free-ing it back to the memory allocator.
// This can have a nontrivial cost that we'd prefer to not pay while holding
// the shard mutex, so we collect the evicted values in a local slice and
// only release them in a defer after dropping the cache mutex.
var obsoleteValuesAlloc [blocksPerMutexAcquisition]*Value
obsoleteValues := obsoleteValuesAlloc[:0]
defer func() {
c.mu.Unlock()
for _, v := range obsoleteValues {
v.release()
}
}()
blocks := c.files.Get(fkey)
if blocks == nil {
// No blocks for this file.
return false
}
// b is the current head of the doubly linked list, and n is the entry after b.
for b, n := blocks, (*entry)(nil); len(obsoleteValues) < cap(obsoleteValues); b = n {
n = b.fileLink.next
obsoleteValues = append(obsoleteValues, c.metaEvict(b))
if b == n {
// b == n represents the case where b was the last entry remaining
// in the doubly linked list, which is why it pointed at itself. So
// no more entries left.
c.checkConsistency()
return false
}
}
// Exhausted blocksPerMutexAcquisition.
return true
}
func (c *shard) Free() {
c.mu.Lock()
defer c.mu.Unlock()
// NB: we use metaDel rather than metaEvict in order to avoid the expensive
// metaCheck call when the "invariants" build tag is specified.
for c.handHot != nil {
e := c.handHot
c.metaDel(c.handHot).release()
e.free()
}
c.blocks.free()
c.files.free()
}
func (c *shard) Reserve(n int) {
c.mu.Lock()
defer c.mu.Unlock()
c.reservedSize += int64(n)
// Changing c.reservedSize will either increase or decrease
// the targetSize. But we want coldTarget to be in the range
// [0, targetSize]. So, if c.targetSize decreases, make sure
// that the coldTarget fits within the limits.
targetSize := c.targetSize()
if c.coldTarget > targetSize {
c.coldTarget = targetSize
}
c.evict()
c.checkConsistency()
}
// Size returns the current space used by the cache.
func (c *shard) Size() int64 {
c.mu.RLock()
size := c.sizeHot + c.sizeCold
c.mu.RUnlock()
return size
}
func (c *shard) targetSize() int64 {
target := c.maxSize - c.reservedSize
// Always return a positive integer for targetSize. This is so that we don't
// end up in an infinite loop in evict(), in cases where reservedSize is
// greater than or equal to maxSize.
if target < 1 {
return 1
}
return target
}
// Add the entry to the cache, returning true if the entry was added and false
// if it would not fit in the cache.
func (c *shard) metaAdd(key key, e *entry) bool {
c.evict()
if e.size > c.targetSize() {
// The entry is larger than the target cache size.
return false
}
c.blocks.Put(key, e)
if entriesGoAllocated {
// Go allocated entries need to be referenced from Go memory. The entries
// map provides that reference.
c.entries[e] = struct{}{}
}
if c.handHot == nil {
// first element
c.handHot = e
c.handCold = e
c.handTest = e
} else {
c.handHot.link(e)
}
if c.handCold == c.handHot {
c.handCold = c.handCold.prev()
}
fkey := key.file()
if fileBlocks := c.files.Get(fkey); fileBlocks == nil {
c.files.Put(fkey, e)
} else {
fileBlocks.linkFile(e)
}
return true
}
// Remove the entry from the cache. This removes the entry from the blocks map,
// the files map, and ensures that hand{Hot,Cold,Test} are not pointing at the
// entry. Returns the deleted value that must be released, if any.
func (c *shard) metaDel(e *entry) (deletedValue *Value) {
if value := e.peekValue(); value != nil {
value.ref.trace("metaDel")
}
// Remove the pointer to the value.
deletedValue = e.val
e.val = nil
c.blocks.Delete(e.key)
if entriesGoAllocated {
// Go allocated entries need to be referenced from Go memory. The entries
// map provides that reference.
delete(c.entries, e)
}
if e == c.handHot {
c.handHot = c.handHot.prev()
}
if e == c.handCold {
c.handCold = c.handCold.prev()
}
if e == c.handTest {
c.handTest = c.handTest.prev()
}
if e.unlink() == e {
// This was the last entry in the cache.
c.handHot = nil
c.handCold = nil
c.handTest = nil
}
fkey := e.key.file()
if next := e.unlinkFile(); e == next {
c.files.Delete(fkey)
} else {
c.files.Put(fkey, next)
}
return deletedValue
}
// Check that the specified entry is not referenced by the cache.
func (c *shard) metaCheck(e *entry) {
if invariants.Enabled {
if _, ok := c.entries[e]; ok {
fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in entries map\n%s",
e, e.key, debug.Stack())
os.Exit(1)
}
if c.blocks.findByValue(e) != nil {
fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in blocks map\n%s\n%s",
e, e.key, &c.blocks, debug.Stack())
os.Exit(1)
}
if c.files.findByValue(e) != nil {
fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in files map\n%s\n%s",
e, e.key, &c.files, debug.Stack())
os.Exit(1)
}
// NB: c.hand{Hot,Cold,Test} are pointers into a single linked list. We
// only have to traverse one of them to check all of them.
var countHot, countCold, countTest int64
var sizeHot, sizeCold, sizeTest int64
for t := c.handHot.next(); t != nil; t = t.next() {
// Recompute count{Hot,Cold,Test} and size{Hot,Cold,Test}.
switch t.ptype {
case etHot:
countHot++
sizeHot += t.size
case etCold:
countCold++
sizeCold += t.size
case etTest:
countTest++
sizeTest += t.size
}
if e == t {
fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in blocks list\n%s",
e, e.key, debug.Stack())
os.Exit(1)
}
if t == c.handHot {
break
}
}
if countHot != c.countHot || countCold != c.countCold || countTest != c.countTest ||
sizeHot != c.sizeHot || sizeCold != c.sizeCold || sizeTest != c.sizeTest {
fmt.Fprintf(os.Stderr, `divergence of Hot,Cold,Test statistics
cache's statistics: hot %d, %d, cold %d, %d, test %d, %d
recalculated statistics: hot %d, %d, cold %d, %d, test %d, %d\n%s`,
c.countHot, c.sizeHot, c.countCold, c.sizeCold, c.countTest, c.sizeTest,
countHot, sizeHot, countCold, sizeCold, countTest, sizeTest,
debug.Stack())
os.Exit(1)
}
}
}
func (c *shard) metaEvict(e *entry) (evictedValue *Value) {
switch e.ptype {
case etHot:
c.sizeHot -= e.size
c.countHot--
case etCold:
c.sizeCold -= e.size
c.countCold--
case etTest:
c.sizeTest -= e.size
c.countTest--
}
evictedValue = c.metaDel(e)
c.metaCheck(e)
e.free()
return evictedValue
}
func (c *shard) evict() {
for c.targetSize() <= c.sizeHot+c.sizeCold && c.handCold != nil {
c.runHandCold(c.countCold, c.sizeCold)
}
}
func (c *shard) runHandCold(countColdDebug, sizeColdDebug int64) {
// countColdDebug and sizeColdDebug should equal c.countCold and
// c.sizeCold. They're parameters only to aid in debugging of
// cockroachdb/cockroach#70154. Since they're parameters, their
// arguments will appear within stack traces should we encounter
// a reproduction.
if c.countCold != countColdDebug || c.sizeCold != sizeColdDebug {
panic(fmt.Sprintf("runHandCold: cold count and size are %d, %d, arguments are %d and %d",
c.countCold, c.sizeCold, countColdDebug, sizeColdDebug))
}
e := c.handCold
if e.ptype == etCold {
if e.referenced.Load() {
e.referenced.Store(false)
e.ptype = etHot
c.sizeCold -= e.size
c.countCold--
c.sizeHot += e.size
c.countHot++
} else {
e.setValue(nil)
e.ptype = etTest
c.sizeCold -= e.size
c.countCold--
c.sizeTest += e.size
c.countTest++
for c.targetSize() < c.sizeTest && c.handTest != nil {
c.runHandTest()
}
}
}
c.handCold = c.handCold.next()
for c.targetSize()-c.coldTarget <= c.sizeHot && c.handHot != nil {
c.runHandHot()
}
}
func (c *shard) runHandHot() {
if c.handHot == c.handTest && c.handTest != nil {
c.runHandTest()
if c.handHot == nil {
return
}
}
e := c.handHot
if e.ptype == etHot {
if e.referenced.Load() {
e.referenced.Store(false)
} else {
e.ptype = etCold
c.sizeHot -= e.size
c.countHot--
c.sizeCold += e.size
c.countCold++
}
}
c.handHot = c.handHot.next()
}
func (c *shard) runHandTest() {
if c.sizeCold > 0 && c.handTest == c.handCold && c.handCold != nil {
// sizeCold is > 0, so assert that countCold == 0. See the
// comment above count{Hot,Cold,Test}.
if c.countCold == 0 {
panic(fmt.Sprintf("pebble: mismatch %d cold size, %d cold count", c.sizeCold, c.countCold))
}
c.runHandCold(c.countCold, c.sizeCold)
if c.handTest == nil {
return
}
}
e := c.handTest
if e.ptype == etTest {
c.sizeTest -= e.size
c.countTest--
c.coldTarget -= e.size
if c.coldTarget < 0 {
c.coldTarget = 0
}
c.metaDel(e).release()
c.metaCheck(e)
e.free()
}
c.handTest = c.handTest.next()
}
// Metrics holds metrics for the cache.
type Metrics struct {
// The number of bytes inuse by the cache.
Size int64
// The count of objects (blocks or tables) in the cache.
Count int64
// The number of cache hits.
Hits int64
// The number of cache misses.
Misses int64
}
// Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is
// used for page replacement
// (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In
// order to provide better concurrency, 4 x NumCPUs shards are created, with
// each shard being given 1/n of the target cache size. The Clock-PRO algorithm
// is run independently on each shard.
//
// Blocks are keyed by an (id, fileNum, offset) triple. The ID is a namespace
// for file numbers and allows a single Cache to be shared between multiple
// Pebble instances. The fileNum and offset refer to an sstable file number and
// the offset of the block within the file. Because sstables are immutable and
// file numbers are never reused, (fileNum,offset) are unique for the lifetime
// of a Pebble instance.
//
// In addition to maintaining a map from (fileNum,offset) to data, each shard
// maintains a map of the cached blocks for a particular fileNum. This allows
// efficient eviction of all of the blocks for a file which is used when an
// sstable is deleted from disk.
//
// # Memory Management
//
// In order to reduce pressure on the Go GC, manual memory management is
// performed for the data stored in the cache. Manual memory management is
// performed by calling into C.{malloc,free} to allocate memory. Cache.Values
// are reference counted and the memory backing a manual value is freed when
// the reference count drops to 0.
//
// Manual memory management brings the possibility of memory leaks. It is
// imperative that every Handle returned by Cache.{Get,Set} is eventually
// released. The "invariants" build tag enables a leak detection facility that
// places a GC finalizer on cache.Value. When the cache.Value finalizer is run,
// if the underlying buffer is still present a leak has occurred. The "tracing"
// build tag enables tracing of cache.Value reference count manipulation and
// eases finding where a leak has occurred. These two facilities are usually
// used in combination by specifying `-tags invariants,tracing`. Note that
// "tracing" produces a significant slowdown, while "invariants" does not.
type Cache struct {
refs atomic.Int64
maxSize int64
idAlloc atomic.Uint64
shards []shard
// Traces recorded by Cache.trace. Used for debugging.
tr struct {
sync.Mutex
msgs []string
}
}
// New creates a new cache of the specified size. Memory for the cache is
// allocated on demand, not during initialization. The cache is created with a
// reference count of 1. Each DB it is associated with adds a reference, so the
// creator of the cache should usually release their reference after the DB is
// created.
//
// c := cache.New(...)
// defer c.Unref()
// d, err := pebble.Open(pebble.Options{Cache: c})
func New(size int64) *Cache {
// How many cache shards should we create?
//
// Note that the probability two processors will try to access the same
// shard at the same time increases superlinearly with the number of
// processors (Eg, consider the brithday problem where each CPU is a person,
// and each shard is a possible birthday).
//
// We could consider growing the number of shards superlinearly, but
// increasing the shard count may reduce the effectiveness of the caching
// algorithm if frequently-accessed blocks are insufficiently distributed
// across shards. If a shard's size is smaller than a single frequently
// scanned sstable, then the shard will be unable to hold the entire
// frequently-scanned table in memory despite other shards still holding
// infrequently accessed blocks.
//
// Experimentally, we've observed contention contributing to tail latencies
// at 2 shards per processor. For now we use 4 shards per processor,
// recognizing this may not be final word.
m := 4 * runtime.GOMAXPROCS(0)
// In tests we can use large CPU machines with small cache sizes and have
// many caches in existence at a time. If sharding into m shards would
// produce too small shards, constrain the number of shards to 4.
const minimumShardSize = 4 << 20 // 4 MiB
if m > 4 && int(size)/m < minimumShardSize {
m = 4
}
return newShards(size, m)
}
func newShards(size int64, shards int) *Cache {
c := &Cache{
maxSize: size,
shards: make([]shard, shards),
}
c.refs.Store(1)
c.idAlloc.Store(1)
c.trace("alloc", c.refs.Load())
for i := range c.shards {
c.shards[i] = shard{
maxSize: size / int64(len(c.shards)),
coldTarget: size / int64(len(c.shards)),
}
if entriesGoAllocated {
c.shards[i].entries = make(map[*entry]struct{})
}
c.shards[i].blocks.init(16)
c.shards[i].files.init(16)
}
// Note: this is a no-op if invariants are disabled or race is enabled.
invariants.SetFinalizer(c, func(obj interface{}) {
c := obj.(*Cache)
if v := c.refs.Load(); v != 0 {
c.tr.Lock()
fmt.Fprintf(os.Stderr,
"pebble: cache (%p) has non-zero reference count: %d\n", c, v)
if len(c.tr.msgs) > 0 {
fmt.Fprintf(os.Stderr, "%s\n", strings.Join(c.tr.msgs, "\n"))
}
c.tr.Unlock()
os.Exit(1)
}
})
return c
}
func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *shard {
if id == 0 {
panic("pebble: 0 cache ID is invalid")
}
// Inlined version of fnv.New64 + Write.
const offset64 = 14695981039346656037
const prime64 = 1099511628211
h := uint64(offset64)
for i := 0; i < 8; i++ {
h *= prime64
h ^= uint64(id & 0xff)
id >>= 8
}
fileNumVal := uint64(fileNum.FileNum())
for i := 0; i < 8; i++ {
h *= prime64
h ^= uint64(fileNumVal) & 0xff
fileNumVal >>= 8
}
for i := 0; i < 8; i++ {
h *= prime64
h ^= uint64(offset & 0xff)
offset >>= 8
}
return &c.shards[h%uint64(len(c.shards))]
}
// Ref adds a reference to the cache. The cache only remains valid as long a
// reference is maintained to it.
func (c *Cache) Ref() {
v := c.refs.Add(1)
if v <= 1 {
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
}
c.trace("ref", v)
}
// Unref releases a reference on the cache.
func (c *Cache) Unref() {
v := c.refs.Add(-1)
c.trace("unref", v)
switch {
case v < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
case v == 0:
for i := range c.shards {
c.shards[i].Free()
}
}
}
// Get retrieves the cache value for the specified file and offset, returning
// nil if no value is present.
func (c *Cache) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
return c.getShard(id, fileNum, offset).Get(id, fileNum, offset)
}
// Set sets the cache value for the specified file and offset, overwriting an
// existing value if present. A Handle is returned which provides faster
// retrieval of the cached value than Get (lock-free and avoidance of the map
// lookup). The value must have been allocated by Cache.Alloc.
func (c *Cache) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
return c.getShard(id, fileNum, offset).Set(id, fileNum, offset, value)
}
// Delete deletes the cached value for the specified file and offset.
func (c *Cache) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
c.getShard(id, fileNum, offset).Delete(id, fileNum, offset)
}
// EvictFile evicts all of the cache values for the specified file.
func (c *Cache) EvictFile(id uint64, fileNum base.DiskFileNum) {
if id == 0 {
panic("pebble: 0 cache ID is invalid")
}
for i := range c.shards {
c.shards[i].EvictFile(id, fileNum)
}
}
// MaxSize returns the max size of the cache.
func (c *Cache) MaxSize() int64 {
return c.maxSize
}
// Size returns the current space used by the cache.
func (c *Cache) Size() int64 {
var size int64
for i := range c.shards {
size += c.shards[i].Size()
}
return size
}
// Alloc allocates a byte slice of the specified size, possibly reusing
// previously allocated but unused memory. The memory backing the value is
// manually managed. The caller MUST either add the value to the cache (via
// Cache.Set), or release the value (via Cache.Free). Failure to do so will
// result in a memory leak.
func Alloc(n int) *Value {
return newValue(n)
}
// Free frees the specified value. The buffer associated with the value will
// possibly be reused, making it invalid to use the buffer after calling
// Free. Do not call Free on a value that has been added to the cache.
func Free(v *Value) {
if n := v.refs(); n > 1 {
panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n))
}
v.release()
}
// Reserve N bytes in the cache. This effectively shrinks the size of the cache
// by N bytes, without actually consuming any memory. The returned closure
// should be invoked to release the reservation.
func (c *Cache) Reserve(n int) func() {
// Round-up the per-shard reservation. Most reservations should be large, so
// this probably doesn't matter in practice.
shardN := (n + len(c.shards) - 1) / len(c.shards)
for i := range c.shards {
c.shards[i].Reserve(shardN)
}
return func() {
if shardN == -1 {
panic("pebble: cache reservation already released")
}
for i := range c.shards {
c.shards[i].Reserve(-shardN)
}
shardN = -1
}
}
// Metrics returns the metrics for the cache.
func (c *Cache) Metrics() Metrics {
var m Metrics
for i := range c.shards {
s := &c.shards[i]
s.mu.RLock()
m.Count += int64(s.blocks.Count())
m.Size += s.sizeHot + s.sizeCold
s.mu.RUnlock()
m.Hits += s.hits.Load()
m.Misses += s.misses.Load()
}
return m
}
// NewID returns a new ID to be used as a namespace for cached file
// blocks.
func (c *Cache) NewID() uint64 {
return c.idAlloc.Add(1)
}

10
vendor/github.com/cockroachdb/pebble/internal/cache/clockpro_normal.go generated vendored Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !tracing
// +build !tracing
package cache
func (c *Cache) trace(_ string, _ int64) {}

20
vendor/github.com/cockroachdb/pebble/internal/cache/clockpro_tracing.go generated vendored Normal file
View File

@@ -0,0 +1,20 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build tracing
// +build tracing
package cache
import (
"fmt"
"runtime/debug"
)
func (c *Cache) trace(msg string, refs int64) {
s := fmt.Sprintf("%s: refs=%d\n%s", msg, refs, debug.Stack())
c.tr.Lock()
c.tr.msgs = append(c.tr.msgs, s)
c.tr.Unlock()
}

155
vendor/github.com/cockroachdb/pebble/internal/cache/entry.go generated vendored Normal file
View File

@@ -0,0 +1,155 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package cache
import "sync/atomic"
type entryType int8
const (
etTest entryType = iota
etCold
etHot
)
func (p entryType) String() string {
switch p {
case etTest:
return "test"
case etCold:
return "cold"
case etHot:
return "hot"
}
return "unknown"
}
// entry holds the metadata for a cache entry. The memory for an entry is
// allocated from manually managed memory.
//
// Using manual memory management for entries is technically a volation of the
// Cgo pointer rules:
//
// https://golang.org/cmd/cgo/#hdr-Passing_pointers
//
// Specifically, Go pointers should not be stored in C allocated memory. The
// reason for this rule is that the Go GC will not look at C allocated memory
// to find pointers to Go objects. If the only reference to a Go object is
// stored in C allocated memory, the object will be reclaimed. The shard field
// of the entry struct points to a Go allocated object, thus the
// violation. What makes this "safe" is that the Cache guarantees that there
// are other pointers to the shard which will keep it alive.
type entry struct {
key key
// The value associated with the entry. The entry holds a reference on the
// value which is maintained by entry.setValue().
val *Value
blockLink struct {
next *entry
prev *entry
}
fileLink struct {
next *entry
prev *entry
}
size int64
ptype entryType
// referenced is atomically set to indicate that this entry has been accessed
// since the last time one of the clock hands swept it.
referenced atomic.Bool
shard *shard
// Reference count for the entry. The entry is freed when the reference count
// drops to zero.
ref refcnt
}
func newEntry(s *shard, key key, size int64) *entry {
e := entryAllocNew()
*e = entry{
key: key,
size: size,
ptype: etCold,
shard: s,
}
e.blockLink.next = e
e.blockLink.prev = e
e.fileLink.next = e
e.fileLink.prev = e
e.ref.init(1)
return e
}
func (e *entry) free() {
e.setValue(nil)
*e = entry{}
entryAllocFree(e)
}
func (e *entry) next() *entry {
if e == nil {
return nil
}
return e.blockLink.next
}
func (e *entry) prev() *entry {
if e == nil {
return nil
}
return e.blockLink.prev
}
func (e *entry) link(s *entry) {
s.blockLink.prev = e.blockLink.prev
s.blockLink.prev.blockLink.next = s
s.blockLink.next = e
s.blockLink.next.blockLink.prev = s
}
func (e *entry) unlink() *entry {
next := e.blockLink.next
e.blockLink.prev.blockLink.next = e.blockLink.next
e.blockLink.next.blockLink.prev = e.blockLink.prev
e.blockLink.prev = e
e.blockLink.next = e
return next
}
func (e *entry) linkFile(s *entry) {
s.fileLink.prev = e.fileLink.prev
s.fileLink.prev.fileLink.next = s
s.fileLink.next = e
s.fileLink.next.fileLink.prev = s
}
func (e *entry) unlinkFile() *entry {
next := e.fileLink.next
e.fileLink.prev.fileLink.next = e.fileLink.next
e.fileLink.next.fileLink.prev = e.fileLink.prev
e.fileLink.prev = e
e.fileLink.next = e
return next
}
func (e *entry) setValue(v *Value) {
if v != nil {
v.acquire()
}
old := e.val
e.val = v
old.release()
}
func (e *entry) peekValue() *Value {
return e.val
}
func (e *entry) acquireValue() *Value {
v := e.val
if v != nil {
v.acquire()
}
return v
}

38
vendor/github.com/cockroachdb/pebble/internal/cache/entry_invariants.go generated vendored Normal file
View File

@@ -0,0 +1,38 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//
//go:build (invariants && !race) || (tracing && !race)
// +build invariants,!race tracing,!race
package cache
import (
"fmt"
"os"
"github.com/cockroachdb/pebble/internal/invariants"
)
// When the "invariants" or "tracing" build tags are enabled, we need to
// allocate entries using the Go allocator so entry.val properly maintains a
// reference to the Value.
const entriesGoAllocated = true
func entryAllocNew() *entry {
e := &entry{}
// Note: this is a no-op if invariants and tracing are disabled or race is
// enabled.
invariants.SetFinalizer(e, func(obj interface{}) {
e := obj.(*entry)
if v := e.ref.refs(); v != 0 {
fmt.Fprintf(os.Stderr, "%p: cache entry has non-zero reference count: %d\n%s",
e, v, e.ref.traces())
os.Exit(1)
}
})
return e
}
func entryAllocFree(e *entry) {
}

103
vendor/github.com/cockroachdb/pebble/internal/cache/entry_normal.go generated vendored Normal file
View File

@@ -0,0 +1,103 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//
//go:build (!invariants && !tracing) || race
// +build !invariants,!tracing race
package cache
import (
"runtime"
"sync"
"unsafe"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manual"
)
const (
entrySize = int(unsafe.Sizeof(entry{}))
entryAllocCacheLimit = 128
// Avoid using runtime.SetFinalizer in race builds as finalizers tickle a bug
// in the Go race detector in go1.15 and earlier versions. This requires that
// entries are Go allocated rather than manually allocated.
//
// If cgo is disabled we need to allocate the entries using the Go allocator
// and is violates the Go GC rules to put Go pointers (such as the entry
// pointer fields) into untyped memory (i.e. a []byte).
entriesGoAllocated = invariants.RaceEnabled || !cgoEnabled
)
var entryAllocPool = sync.Pool{
New: func() interface{} {
return newEntryAllocCache()
},
}
func entryAllocNew() *entry {
a := entryAllocPool.Get().(*entryAllocCache)
e := a.alloc()
entryAllocPool.Put(a)
return e
}
func entryAllocFree(e *entry) {
a := entryAllocPool.Get().(*entryAllocCache)
a.free(e)
entryAllocPool.Put(a)
}
type entryAllocCache struct {
entries []*entry
}
func newEntryAllocCache() *entryAllocCache {
c := &entryAllocCache{}
if !entriesGoAllocated {
// Note the use of a "real" finalizer here (as opposed to a build tag-gated
// no-op finalizer). Without the finalizer, objects released from the pool
// and subsequently GC'd by the Go runtime would fail to have their manually
// allocated memory freed, which results in a memory leak.
// lint:ignore SetFinalizer
runtime.SetFinalizer(c, freeEntryAllocCache)
}
return c
}
func freeEntryAllocCache(obj interface{}) {
c := obj.(*entryAllocCache)
for i, e := range c.entries {
c.dealloc(e)
c.entries[i] = nil
}
}
func (c *entryAllocCache) alloc() *entry {
n := len(c.entries)
if n == 0 {
if entriesGoAllocated {
return &entry{}
}
b := manual.New(entrySize)
return (*entry)(unsafe.Pointer(&b[0]))
}
e := c.entries[n-1]
c.entries = c.entries[:n-1]
return e
}
func (c *entryAllocCache) dealloc(e *entry) {
if !entriesGoAllocated {
buf := (*[manual.MaxArrayLen]byte)(unsafe.Pointer(e))[:entrySize:entrySize]
manual.Free(buf)
}
}
func (c *entryAllocCache) free(e *entry) {
if len(c.entries) == entryAllocCacheLimit {
c.dealloc(e)
return
}
c.entries = append(c.entries, e)
}

59
vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_normal.go generated vendored Normal file
View File

@@ -0,0 +1,59 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !tracing
// +build !tracing
package cache
import (
"fmt"
"sync/atomic"
"github.com/cockroachdb/redact"
)
// refcnt provides an atomic reference count. This version is used when the
// "tracing" build tag is not enabled. See refcnt_tracing.go for the "tracing"
// enabled version.
type refcnt struct {
val atomic.Int32
}
// initialize the reference count to the specified value.
func (v *refcnt) init(val int32) {
v.val.Store(val)
}
func (v *refcnt) refs() int32 {
return v.val.Load()
}
func (v *refcnt) acquire() {
switch v := v.val.Add(1); {
case v <= 1:
panic(redact.Safe(fmt.Sprintf("pebble: inconsistent reference count: %d", v)))
}
}
func (v *refcnt) release() bool {
switch v := v.val.Add(-1); {
case v < 0:
panic(redact.Safe(fmt.Sprintf("pebble: inconsistent reference count: %d", v)))
case v == 0:
return true
default:
return false
}
}
func (v *refcnt) trace(msg string) {
}
func (v *refcnt) traces() string {
return ""
}
// Silence unused warning.
var _ = (*refcnt)(nil).traces

66
vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_tracing.go generated vendored Normal file
View File

@@ -0,0 +1,66 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build tracing
// +build tracing
package cache
import (
"fmt"
"runtime/debug"
"strings"
"sync"
"sync/atomic"
)
// refcnt provides an atomic reference count, along with a tracing facility for
// debugging logic errors in manipulating the reference count. This version is
// used when the "tracing" build tag is enabled.
type refcnt struct {
val atomic.Int32
sync.Mutex
msgs []string
}
func (v *refcnt) init(val int32) {
v.val.Store(val)
v.trace("init")
}
func (v *refcnt) refs() int32 {
return v.val.Load()
}
func (v *refcnt) acquire() {
switch n := v.val.Add(1); {
case n <= 1:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))
}
v.trace("acquire")
}
func (v *refcnt) release() bool {
n := v.val.Add(-1)
switch {
case n < 0:
panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))
}
v.trace("release")
return n == 0
}
func (v *refcnt) trace(msg string) {
s := fmt.Sprintf("%s: refs=%d\n%s", msg, v.refs(), debug.Stack())
v.Lock()
v.msgs = append(v.msgs, s)
v.Unlock()
}
func (v *refcnt) traces() string {
v.Lock()
s := strings.Join(v.msgs, "\n")
v.Unlock()
return s
}

320
vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go generated vendored Normal file
View File

@@ -0,0 +1,320 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package cache
import (
"fmt"
"math/bits"
"os"
"runtime/debug"
"strings"
"time"
"unsafe"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manual"
)
var hashSeed = uint64(time.Now().UnixNano())
// Fibonacci hash: https://probablydance.com/2018/06/16/fibonacci-hashing-the-optimization-that-the-world-forgot-or-a-better-alternative-to-integer-modulo/
func robinHoodHash(k key, shift uint32) uint32 {
const m = 11400714819323198485
h := hashSeed
h ^= k.id * m
h ^= uint64(k.fileNum.FileNum()) * m
h ^= k.offset * m
return uint32(h >> shift)
}
type robinHoodEntry struct {
key key
// Note that value may point to a Go allocated object (if the "invariants"
// build tag was specified), even though the memory for the entry itself is
// manually managed. This is technically a volation of the Cgo pointer rules:
//
// https://golang.org/cmd/cgo/#hdr-Passing_pointers
//
// Specifically, Go pointers should not be stored in C allocated memory. The
// reason for this rule is that the Go GC will not look at C allocated memory
// to find pointers to Go objects. If the only reference to a Go object is
// stored in C allocated memory, the object will be reclaimed. What makes
// this "safe" is that the Cache guarantees that there are other pointers to
// the entry and shard which will keep them alive. In particular, every Go
// allocated entry in the cache is referenced by the shard.entries map. And
// every shard is referenced by the Cache.shards map.
value *entry
// The distance the entry is from its desired position.
dist uint32
}
type robinHoodEntries struct {
ptr unsafe.Pointer
len uint32
}
func newRobinHoodEntries(n uint32) robinHoodEntries {
size := uintptr(n) * unsafe.Sizeof(robinHoodEntry{})
return robinHoodEntries{
ptr: unsafe.Pointer(&(manual.New(int(size)))[0]),
len: n,
}
}
func (e robinHoodEntries) at(i uint32) *robinHoodEntry {
return (*robinHoodEntry)(unsafe.Pointer(uintptr(e.ptr) +
uintptr(i)*unsafe.Sizeof(robinHoodEntry{})))
}
func (e robinHoodEntries) free() {
size := uintptr(e.len) * unsafe.Sizeof(robinHoodEntry{})
buf := (*[manual.MaxArrayLen]byte)(e.ptr)[:size:size]
manual.Free(buf)
}
// robinHoodMap is an implementation of Robin Hood hashing. Robin Hood hashing
// is an open-address hash table using linear probing. The twist is that the
// linear probe distance is reduced by moving existing entries when inserting
// and deleting. This is accomplished by keeping track of how far an entry is
// from its "desired" slot (hash of key modulo number of slots). During
// insertion, if the new entry being inserted is farther from its desired slot
// than the target entry, we swap the target and new entry. This effectively
// steals from the "rich" target entry and gives to the "poor" new entry (thus
// the origin of the name).
//
// An extension over the base Robin Hood hashing idea comes from
// https://probablydance.com/2017/02/26/i-wrote-the-fastest-hashtable/. A cap
// is placed on the max distance an entry can be from its desired slot. When
// this threshold is reached during insertion, the size of the table is doubled
// and insertion is restarted. Additionally, the entries slice is given "max
// dist" extra entries on the end. The very last entry in the entries slice is
// never used and acts as a sentinel which terminates loops. The previous
// maxDist-1 entries act as the extra entries. For example, if the size of the
// table is 2, maxDist is computed as 4 and the actual size of the entry slice
// is 6.
//
// +---+---+---+---+---+---+
// | 0 | 1 | 2 | 3 | 4 | 5 |
// +---+---+---+---+---+---+
// ^
// size
//
// In this scenario, the target entry for a key will always be in the range
// [0,1]. Valid entries may reside in the range [0,4] due to the linear probing
// of up to maxDist entries. The entry at index 5 will never contain a value,
// and instead acts as a sentinel (its distance is always 0). The max distance
// threshold is set to log2(num-entries). This ensures that retrieval is O(log
// N), though note that N is the number of total entries, not the count of
// valid entries.
//
// Deletion is implemented via the backward shift delete mechanism instead of
// tombstones. This preserves the performance of the table in the presence of
// deletions. See
// http://codecapsule.com/2013/11/17/robin-hood-hashing-backward-shift-deletion
// for details.
type robinHoodMap struct {
entries robinHoodEntries
size uint32
shift uint32
count uint32
maxDist uint32
}
func maxDistForSize(size uint32) uint32 {
desired := uint32(bits.Len32(size))
if desired < 4 {
desired = 4
}
return desired
}
func newRobinHoodMap(initialCapacity int) *robinHoodMap {
m := &robinHoodMap{}
m.init(initialCapacity)
// Note: this is a no-op if invariants are disabled or race is enabled.
invariants.SetFinalizer(m, func(obj interface{}) {
m := obj.(*robinHoodMap)
if m.entries.ptr != nil {
fmt.Fprintf(os.Stderr, "%p: robin-hood map not freed\n", m)
os.Exit(1)
}
})
return m
}
func (m *robinHoodMap) init(initialCapacity int) {
if initialCapacity < 1 {
initialCapacity = 1
}
targetSize := 1 << (uint(bits.Len(uint(2*initialCapacity-1))) - 1)
m.rehash(uint32(targetSize))
}
func (m *robinHoodMap) free() {
if m.entries.ptr != nil {
m.entries.free()
m.entries.ptr = nil
}
}
func (m *robinHoodMap) rehash(size uint32) {
oldEntries := m.entries
m.size = size
m.shift = uint32(64 - bits.Len32(m.size-1))
m.maxDist = maxDistForSize(size)
m.entries = newRobinHoodEntries(size + m.maxDist)
m.count = 0
for i := uint32(0); i < oldEntries.len; i++ {
e := oldEntries.at(i)
if e.value != nil {
m.Put(e.key, e.value)
}
}
if oldEntries.ptr != nil {
oldEntries.free()
}
}
// Find an entry containing the specified value. This is intended to be used
// from debug and test code.
func (m *robinHoodMap) findByValue(v *entry) *robinHoodEntry {
for i := uint32(0); i < m.entries.len; i++ {
e := m.entries.at(i)
if e.value == v {
return e
}
}
return nil
}
func (m *robinHoodMap) Count() int {
return int(m.count)
}
func (m *robinHoodMap) Put(k key, v *entry) {
maybeExists := true
n := robinHoodEntry{key: k, value: v, dist: 0}
for i := robinHoodHash(k, m.shift); ; i++ {
e := m.entries.at(i)
if maybeExists && k == e.key {
// Entry already exists: overwrite.
e.value = n.value
m.checkEntry(i)
return
}
if e.value == nil {
// Found an empty entry: insert here.
*e = n
m.count++
m.checkEntry(i)
return
}
if e.dist < n.dist {
// Swap the new entry with the current entry because the current is
// rich. We then continue to loop, looking for a new location for the
// current entry. Note that this is also the not-found condition for
// retrieval, which means that "k" is not present in the map. See Get().
n, *e = *e, n
m.checkEntry(i)
maybeExists = false
}
// The new entry gradually moves away from its ideal position.
n.dist++
// If we've reached the max distance threshold, grow the table and restart
// the insertion.
if n.dist == m.maxDist {
m.rehash(2 * m.size)
i = robinHoodHash(n.key, m.shift) - 1
n.dist = 0
maybeExists = false
}
}
}
func (m *robinHoodMap) Get(k key) *entry {
var dist uint32
for i := robinHoodHash(k, m.shift); ; i++ {
e := m.entries.at(i)
if k == e.key {
// Found.
return e.value
}
if e.dist < dist {
// Not found.
return nil
}
dist++
}
}
func (m *robinHoodMap) Delete(k key) {
var dist uint32
for i := robinHoodHash(k, m.shift); ; i++ {
e := m.entries.at(i)
if k == e.key {
m.checkEntry(i)
// We found the entry to delete. Shift the following entries backwards
// until the next empty value or entry with a zero distance. Note that
// empty values are guaranteed to have "dist == 0".
m.count--
for j := i + 1; ; j++ {
t := m.entries.at(j)
if t.dist == 0 {
*e = robinHoodEntry{}
return
}
e.key = t.key
e.value = t.value
e.dist = t.dist - 1
e = t
m.checkEntry(j)
}
}
if dist > e.dist {
// Not found.
return
}
dist++
}
}
func (m *robinHoodMap) checkEntry(i uint32) {
if invariants.Enabled {
e := m.entries.at(i)
if e.value != nil {
pos := robinHoodHash(e.key, m.shift)
if (uint32(i) - pos) != e.dist {
fmt.Fprintf(os.Stderr, "%d: invalid dist=%d, expected %d: %s\n%s",
i, e.dist, uint32(i)-pos, e.key, debug.Stack())
os.Exit(1)
}
if e.dist > m.maxDist {
fmt.Fprintf(os.Stderr, "%d: invalid dist=%d > maxDist=%d: %s\n%s",
i, e.dist, m.maxDist, e.key, debug.Stack())
os.Exit(1)
}
}
}
}
func (m *robinHoodMap) String() string {
var buf strings.Builder
fmt.Fprintf(&buf, "count: %d\n", m.count)
for i := uint32(0); i < m.entries.len; i++ {
e := m.entries.at(i)
if e.value != nil {
fmt.Fprintf(&buf, "%d: [%s,%p,%d]\n", i, e.key, e.value, e.dist)
}
}
return buf.String()
}

46
vendor/github.com/cockroachdb/pebble/internal/cache/value.go generated vendored Normal file
View File

@@ -0,0 +1,46 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package cache
// Value holds a reference counted immutable value.
type Value struct {
buf []byte
// Reference count for the value. The value is freed when the reference count
// drops to zero.
ref refcnt
}
// Buf returns the buffer associated with the value. The contents of the buffer
// should not be changed once the value has been added to the cache. Instead, a
// new Value should be created and added to the cache to replace the existing
// value.
func (v *Value) Buf() []byte {
if v == nil {
return nil
}
return v.buf
}
// Truncate the buffer to the specified length. The buffer length should not be
// changed once the value has been added to the cache as there may be
// concurrent readers of the Value. Instead, a new Value should be created and
// added to the cache to replace the existing value.
func (v *Value) Truncate(n int) {
v.buf = v.buf[:n]
}
func (v *Value) refs() int32 {
return v.ref.refs()
}
func (v *Value) acquire() {
v.ref.acquire()
}
func (v *Value) release() {
if v != nil && v.ref.release() {
v.free()
}
}

55
vendor/github.com/cockroachdb/pebble/internal/cache/value_invariants.go generated vendored Normal file
View File

@@ -0,0 +1,55 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build (invariants && !race) || (tracing && !race)
// +build invariants,!race tracing,!race
package cache
import (
"fmt"
"os"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manual"
)
// newValue creates a Value with a manually managed buffer of size n.
//
// This definition of newValue is used when either the "invariants" or
// "tracing" build tags are specified. It hooks up a finalizer to the returned
// Value that checks for memory leaks when the GC determines the Value is no
// longer reachable.
func newValue(n int) *Value {
if n == 0 {
return nil
}
b := manual.New(n)
v := &Value{buf: b}
v.ref.init(1)
// Note: this is a no-op if invariants and tracing are disabled or race is
// enabled.
invariants.SetFinalizer(v, func(obj interface{}) {
v := obj.(*Value)
if v.buf != nil {
fmt.Fprintf(os.Stderr, "%p: cache value was not freed: refs=%d\n%s",
v, v.refs(), v.ref.traces())
os.Exit(1)
}
})
return v
}
func (v *Value) free() {
// When "invariants" are enabled set the value contents to 0xff in order to
// cache use-after-free bugs.
for i := range v.buf {
v.buf[i] = 0xff
}
manual.Free(v.buf)
// Setting Value.buf to nil is needed for correctness of the leak checking
// that is performed when the "invariants" or "tracing" build tags are
// enabled.
v.buf = nil
}

57
vendor/github.com/cockroachdb/pebble/internal/cache/value_normal.go generated vendored Normal file
View File

@@ -0,0 +1,57 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build (!invariants && !tracing) || race
// +build !invariants,!tracing race
package cache
import (
"unsafe"
"github.com/cockroachdb/pebble/internal/manual"
)
const valueSize = int(unsafe.Sizeof(Value{}))
func newValue(n int) *Value {
if n == 0 {
return nil
}
if !cgoEnabled {
// If Cgo is disabled then all memory is allocated from the Go heap and we
// can't play the trick below to combine the Value and buffer allocation.
v := &Value{buf: make([]byte, n)}
v.ref.init(1)
return v
}
// When we're not performing leak detection, the lifetime of the returned
// Value is exactly the lifetime of the backing buffer and we can manually
// allocate both.
//
// TODO(peter): It may be better to separate the allocation of the value and
// the buffer in order to reduce internal fragmentation in malloc. If the
// buffer is right at a power of 2, adding valueSize might push the
// allocation over into the next larger size.
b := manual.New(valueSize + n)
v := (*Value)(unsafe.Pointer(&b[0]))
v.buf = b[valueSize:]
v.ref.init(1)
return v
}
func (v *Value) free() {
if !cgoEnabled {
return
}
// When we're not performing leak detection, the Value and buffer were
// allocated contiguously.
n := valueSize + cap(v.buf)
buf := (*[manual.MaxArrayLen]byte)(unsafe.Pointer(v))[:n:n]
v.buf = nil
manual.Free(buf)
}

17
vendor/github.com/cockroachdb/pebble/internal/constants/constants.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package constants
const (
// oneIf64Bit is 1 on 64-bit platforms and 0 on 32-bit platforms.
oneIf64Bit = ^uint(0) >> 63
// MaxUint32OrInt returns min(MaxUint32, MaxInt), i.e
// - MaxUint32 on 64-bit platforms;
// - MaxInt on 32-bit platforms.
// It is used when slices are limited to Uint32 on 64-bit platforms (the
// length limit for slices is naturally MaxInt on 32-bit platforms).
MaxUint32OrInt = (1<<31)<<oneIf64Bit - 1
)

42
vendor/github.com/cockroachdb/pebble/internal/crc/crc.go generated vendored Normal file
View File

@@ -0,0 +1,42 @@
// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
// Package crc implements the checksum algorithm used throughout pebble.
//
// The algorithm is CRC-32 with Castagnoli's polynomial, followed by a bit
// rotation and an additional delta. The additional processing is to lessen the
// probability of arbitrary key/value data coincidentally containing bytes that
// look like a checksum.
//
// To calculate the uint32 checksum of some data:
//
// var u uint32 = crc.New(data).Value()
//
// In pebble, the uint32 value is then stored in little-endian format.
package crc // import "github.com/cockroachdb/pebble/internal/crc"
import "hash/crc32"
var table = crc32.MakeTable(crc32.Castagnoli)
// CRC is a small convenience wrapper for computing the CRC32 checksum used by
// pebble. This is the same algorithm as used by RocksDB.
type CRC uint32
// New returns the result of adding the bytes to the zero-value CRC.
func New(b []byte) CRC {
return CRC(0).Update(b)
}
// Update returns the result of adding the bytes to the CRC.
func (c CRC) Update(b []byte) CRC {
return CRC(crc32.Update(uint32(c), table, b))
}
// Value returns the cooked CRC value. The additional processing is to lessen
// the probability of arbitrary key/value data coincidentally containing bytes
// that look like a checksum.
func (c CRC) Value() uint32 {
return uint32(c>>15|c<<17) + 0xa282ead8
}

17
vendor/github.com/cockroachdb/pebble/internal/fastrand/fastrand.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package fastrand
import _ "unsafe" // required by go:linkname
// Uint32 returns a lock free uint32 value.
//
//go:linkname Uint32 runtime.fastrand
func Uint32() uint32
// Uint32n returns a lock free uint32 value in the interval [0, n).
//
//go:linkname Uint32n runtime.fastrandn
func Uint32n(n uint32) uint32

68
vendor/github.com/cockroachdb/pebble/internal/humanize/humanize.go generated vendored Normal file
View File

@@ -0,0 +1,68 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package humanize
import (
"fmt"
"math"
"github.com/cockroachdb/redact"
)
func logn(n, b float64) float64 {
return math.Log(n) / math.Log(b)
}
func humanate(s uint64, base float64, suffixes []string) string {
if s < 10 {
return fmt.Sprintf("%d%s", s, suffixes[0])
}
e := math.Floor(logn(float64(s), base))
suffix := suffixes[int(e)]
val := math.Floor(float64(s)/math.Pow(base, e)*10+0.5) / 10
f := "%.0f%s"
if val < 10 {
f = "%.1f%s"
}
return fmt.Sprintf(f, val, suffix)
}
type config struct {
base float64
suffix []string
}
// Bytes produces human readable representations of byte values in IEC units.
var Bytes = config{1024, []string{"B", "KB", "MB", "GB", "TB", "PB", "EB"}}
// Count produces human readable representations of unitless values in SI units.
var Count = config{1000, []string{"", "K", "M", "G", "T", "P", "E"}}
// Int64 produces a human readable representation of the value.
func (c *config) Int64(s int64) FormattedString {
if s < 0 {
return FormattedString("-" + humanate(uint64(-s), c.base, c.suffix))
}
return FormattedString(humanate(uint64(s), c.base, c.suffix))
}
// Uint64 produces a human readable representation of the value.
func (c *config) Uint64(s uint64) FormattedString {
return FormattedString(humanate(s, c.base, c.suffix))
}
// FormattedString represents a human readable representation of a value. It
// implements the redact.SafeValue interface to signal that it represents a
// a string that does not need to be redacted.
type FormattedString string
var _ redact.SafeValue = FormattedString("")
// SafeValue implements redact.SafeValue.
func (fs FormattedString) SafeValue() {}
// String implements fmt.Stringer.
func (fs FormattedString) String() string { return string(fs) }

27
vendor/github.com/cockroachdb/pebble/internal/intern/intern.go generated vendored Normal file
View File

@@ -0,0 +1,27 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package intern
import "sync"
var pool = sync.Pool{
New: func() interface{} {
return make(map[string]string)
},
}
// Bytes returns b converted to a string, interned.
func Bytes(b []byte) string {
m := pool.Get().(map[string]string)
c, ok := m[string(b)]
if ok {
pool.Put(m)
return c
}
s := string(b)
m[s] = s
pool.Put(m)
return s
}

14
vendor/github.com/cockroachdb/pebble/internal/invariants/finalizer_off.go generated vendored Normal file
View File

@@ -0,0 +1,14 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build (!invariants && !tracing) || race
// +build !invariants,!tracing race
package invariants
// SetFinalizer is a wrapper around runtime.SetFinalizer that is a no-op under
// race builds or if neither the invariants or tracing build tags are
// specified.
func SetFinalizer(obj, finalizer interface{}) {
}

17
vendor/github.com/cockroachdb/pebble/internal/invariants/finalizer_on.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build (invariants && !race) || (tracing && !race)
// +build invariants,!race tracing,!race
package invariants
import "runtime"
// SetFinalizer is a wrapper around runtime.SetFinalizer that is a no-op under
// race builds or if neither the invariants or tracing build tags are
// specified.
func SetFinalizer(obj, finalizer interface{}) {
runtime.SetFinalizer(obj, finalizer)
}

11
vendor/github.com/cockroachdb/pebble/internal/invariants/off.go generated vendored Normal file
View File

@@ -0,0 +1,11 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !invariants && !race
// +build !invariants,!race
package invariants
// Enabled is true if we were built with the "invariants" or "race" build tags.
const Enabled = false

11
vendor/github.com/cockroachdb/pebble/internal/invariants/on.go generated vendored Normal file
View File

@@ -0,0 +1,11 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build invariants || race
// +build invariants race
package invariants
// Enabled is true if we were built with the "invariants" or "race" build tags.
const Enabled = true

11
vendor/github.com/cockroachdb/pebble/internal/invariants/race_off.go generated vendored Normal file
View File

@@ -0,0 +1,11 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !race
// +build !race
package invariants
// RaceEnabled is true if we were built with the "race" build tag.
const RaceEnabled = false

11
vendor/github.com/cockroachdb/pebble/internal/invariants/race_on.go generated vendored Normal file
View File

@@ -0,0 +1,11 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build race
// +build race
package invariants
// RaceEnabled is true if we were built with the "race" build tag.
const RaceEnabled = true

268
vendor/github.com/cockroachdb/pebble/internal/keyspan/bounded.go generated vendored Normal file
View File

@@ -0,0 +1,268 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// TODO(jackson): Consider removing this type and adding bounds enforcement
// directly to the MergingIter. This type is probably too lightweight to warrant
// its own type, but for now we implement it separately for expediency.
// boundedIterPos records the position of the BoundedIter relative to the
// underlying iterator's position. It's used to avoid Next/Prev-ing the iterator
// if there can't possibly be another span within bounds, because the current
// span overlaps the bound.
//
// Imagine bounds [a,c) and an iterator that seeks to a span [b,d). The span
// [b,d) overlaps some portion of the iterator bounds, so the iterator must
// return it. If the iterator is subsequently Nexted, Next can tell that the
// iterator is exhausted without advancing the underlying iterator because the
// current span's end bound of d is ≥ the upper bound of c. In this case, the
// bounded iterator returns nil and records i.pos as posAtUpperLimit to remember
// that the underlying iterator position does not match the current BoundedIter
// position.
type boundedIterPos int8
const (
posAtLowerLimit boundedIterPos = -1
posAtIterSpan boundedIterPos = 0
posAtUpperLimit boundedIterPos = +1
)
// BoundedIter implements FragmentIterator and enforces bounds.
//
// Like the point InternalIterator interface, the bounded iterator's forward
// positioning routines (SeekGE, First, and Next) only check the upper bound.
// The reverse positioning routines (SeekLT, Last, and Prev) only check the
// lower bound. It is up to the caller to ensure that the forward positioning
// routines respect the lower bound and the reverse positioning routines respect
// the upper bound (i.e. calling SeekGE instead of First if there is a lower
// bound, and SeekLT instead of Last if there is an upper bound).
//
// When the hasPrefix parameter indicates that the iterator is in prefix
// iteration mode, BoundedIter elides any spans that do not overlap with the
// prefix's keyspace. In prefix iteration mode, reverse iteration is disallowed,
// except for an initial SeekLT with a seek key greater than or equal to the
// prefix. In prefix iteration mode, the first seek must position the iterator
// at or immediately before the first fragment covering a key greater than or
// equal to the prefix.
type BoundedIter struct {
iter FragmentIterator
iterSpan *Span
cmp base.Compare
split base.Split
lower []byte
upper []byte
hasPrefix *bool
prefix *[]byte
pos boundedIterPos
}
// Init initializes the bounded iterator.
//
// In addition to the iterator bounds, Init takes pointers to a boolean
// indicating whether the iterator is in prefix iteration mode and the prefix
// key if it is. This is used to exclude spans that are outside the iteration
// prefix.
//
// hasPrefix and prefix are allowed to be nil, however if hasPrefix != nil,
// prefix must also not be nil.
func (i *BoundedIter) Init(
cmp base.Compare,
split base.Split,
iter FragmentIterator,
lower, upper []byte,
hasPrefix *bool,
prefix *[]byte,
) {
*i = BoundedIter{
iter: iter,
cmp: cmp,
split: split,
lower: lower,
upper: upper,
hasPrefix: hasPrefix,
prefix: prefix,
}
}
var _ FragmentIterator = (*BoundedIter)(nil)
// Seek calls.
//
// Seek calls check iterator bounds in the direction of the seek. Additionally,
// if the iterator is in prefix iteration mode, seek calls check both start and
// end bounds against the prefix's bounds. We check both bounds for defense in
// depth. This optimization has been a source of various bugs due to various
// other prefix iteration optimizations that can result in seek keys that don't
// respect the prefix bounds.
// SeekGE implements FragmentIterator.
func (i *BoundedIter) SeekGE(key []byte) *Span {
s := i.iter.SeekGE(key)
s = i.checkPrefixSpanStart(s)
s = i.checkPrefixSpanEnd(s)
return i.checkForwardBound(s)
}
// SeekLT implements FragmentIterator.
func (i *BoundedIter) SeekLT(key []byte) *Span {
s := i.iter.SeekLT(key)
s = i.checkPrefixSpanStart(s)
s = i.checkPrefixSpanEnd(s)
return i.checkBackwardBound(s)
}
// First implements FragmentIterator.
func (i *BoundedIter) First() *Span {
s := i.iter.First()
s = i.checkPrefixSpanStart(s)
return i.checkForwardBound(s)
}
// Last implements FragmentIterator.
func (i *BoundedIter) Last() *Span {
s := i.iter.Last()
s = i.checkPrefixSpanEnd(s)
return i.checkBackwardBound(s)
}
// Next implements FragmentIterator.
func (i *BoundedIter) Next() *Span {
switch i.pos {
case posAtLowerLimit:
// The BoundedIter had previously returned nil, because it knew from
// i.iterSpan's bounds that there was no previous span. To Next, we only
// need to return the current iter span and reset i.pos to reflect that
// we're no longer positioned at the limit.
i.pos = posAtIterSpan
return i.iterSpan
case posAtIterSpan:
// If the span at the underlying iterator position extends to or beyond the
// upper bound, we can avoid advancing because the next span is necessarily
// out of bounds.
if i.iterSpan != nil && i.upper != nil && i.cmp(i.iterSpan.End, i.upper) >= 0 {
i.pos = posAtUpperLimit
return nil
}
// Similarly, if the span extends to the next prefix and we're in prefix
// iteration mode, we can avoid advancing.
if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix {
ei := i.split(i.iterSpan.End)
if i.cmp(i.iterSpan.End[:ei], *i.prefix) > 0 {
i.pos = posAtUpperLimit
return nil
}
}
return i.checkForwardBound(i.checkPrefixSpanStart(i.iter.Next()))
case posAtUpperLimit:
// Already exhausted.
return nil
default:
panic("unreachable")
}
}
// Prev implements FragmentIterator.
func (i *BoundedIter) Prev() *Span {
switch i.pos {
case posAtLowerLimit:
// Already exhausted.
return nil
case posAtIterSpan:
// If the span at the underlying iterator position extends to or beyond
// the lower bound, we can avoid advancing because the previous span is
// necessarily out of bounds.
if i.iterSpan != nil && i.lower != nil && i.cmp(i.iterSpan.Start, i.lower) <= 0 {
i.pos = posAtLowerLimit
return nil
}
// Similarly, if the span extends to or beyond the current prefix and
// we're in prefix iteration mode, we can avoid advancing.
if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix {
si := i.split(i.iterSpan.Start)
if i.cmp(i.iterSpan.Start[:si], *i.prefix) < 0 {
i.pos = posAtLowerLimit
return nil
}
}
return i.checkBackwardBound(i.checkPrefixSpanEnd(i.iter.Prev()))
case posAtUpperLimit:
// The BoundedIter had previously returned nil, because it knew from
// i.iterSpan's bounds that there was no next span. To Prev, we only
// need to return the current iter span and reset i.pos to reflect that
// we're no longer positioned at the limit.
i.pos = posAtIterSpan
return i.iterSpan
default:
panic("unreachable")
}
}
// Error implements FragmentIterator.
func (i *BoundedIter) Error() error {
return i.iter.Error()
}
// Close implements FragmentIterator.
func (i *BoundedIter) Close() error {
return i.iter.Close()
}
// SetBounds modifies the FragmentIterator's bounds.
func (i *BoundedIter) SetBounds(lower, upper []byte) {
i.lower, i.upper = lower, upper
}
func (i *BoundedIter) checkPrefixSpanStart(span *Span) *Span {
// Compare to the prefix's bounds, if in prefix iteration mode.
if span != nil && i.hasPrefix != nil && *i.hasPrefix {
si := i.split(span.Start)
if i.cmp(span.Start[:si], *i.prefix) > 0 {
// This span starts at a prefix that sorts after our current prefix.
span = nil
}
}
return span
}
// checkForwardBound enforces the upper bound, returning nil if the provided
// span is wholly outside the upper bound. It also updates i.pos and i.iterSpan
// to reflect the new iterator position.
func (i *BoundedIter) checkForwardBound(span *Span) *Span {
// Compare to the upper bound.
if span != nil && i.upper != nil && i.cmp(span.Start, i.upper) >= 0 {
span = nil
}
i.iterSpan = span
if i.pos != posAtIterSpan {
i.pos = posAtIterSpan
}
return span
}
func (i *BoundedIter) checkPrefixSpanEnd(span *Span) *Span {
// Compare to the prefix's bounds, if in prefix iteration mode.
if span != nil && i.hasPrefix != nil && *i.hasPrefix && i.cmp(span.End, *i.prefix) <= 0 {
// This span ends before the current prefix.
span = nil
}
return span
}
// checkBackward enforces the lower bound, returning nil if the provided span is
// wholly outside the lower bound. It also updates i.pos and i.iterSpan to
// reflect the new iterator position.
func (i *BoundedIter) checkBackwardBound(span *Span) *Span {
// Compare to the lower bound.
if span != nil && i.lower != nil && i.cmp(span.End, i.lower) <= 0 {
span = nil
}
i.iterSpan = span
if i.pos != posAtIterSpan {
i.pos = posAtIterSpan
}
return span
}

505
vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go generated vendored Normal file
View File

@@ -0,0 +1,505 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"bytes"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/bytealloc"
"github.com/cockroachdb/pebble/internal/invariants"
)
// bufferReuseMaxCapacity is the maximum capacity of a DefragmentingIter buffer
// that DefragmentingIter will reuse. Buffers larger than this will be
// discarded and reallocated as necessary.
const bufferReuseMaxCapacity = 10 << 10 // 10 KB
// keysReuseMaxCapacity is the maximum capacity of a []keyspan.Key buffer that
// DefragmentingIter will reuse. Buffers larger than this will be discarded and
// reallocated as necessary.
const keysReuseMaxCapacity = 100
// DefragmentMethod configures the defragmentation performed by the
// DefragmentingIter.
type DefragmentMethod interface {
// ShouldDefragment takes two abutting spans and returns whether the two
// spans should be combined into a single, defragmented Span.
ShouldDefragment(equal base.Equal, left, right *Span) bool
}
// The DefragmentMethodFunc type is an adapter to allow the use of ordinary
// functions as DefragmentMethods. If f is a function with the appropriate
// signature, DefragmentMethodFunc(f) is a DefragmentMethod that calls f.
type DefragmentMethodFunc func(equal base.Equal, left, right *Span) bool
// ShouldDefragment calls f(equal, left, right).
func (f DefragmentMethodFunc) ShouldDefragment(equal base.Equal, left, right *Span) bool {
return f(equal, left, right)
}
// DefragmentInternal configures a DefragmentingIter to defragment spans
// only if they have identical keys. It requires spans' keys to be sorted in
// trailer descending order.
//
// This defragmenting method is intended for use in compactions that may see
// internal range keys fragments that may now be joined, because the state that
// required their fragmentation has been dropped.
var DefragmentInternal DefragmentMethod = DefragmentMethodFunc(func(equal base.Equal, a, b *Span) bool {
if a.KeysOrder != ByTrailerDesc || b.KeysOrder != ByTrailerDesc {
panic("pebble: span keys unexpectedly not in trailer descending order")
}
if len(a.Keys) != len(b.Keys) {
return false
}
for i := range a.Keys {
if a.Keys[i].Trailer != b.Keys[i].Trailer {
return false
}
if !equal(a.Keys[i].Suffix, b.Keys[i].Suffix) {
return false
}
if !bytes.Equal(a.Keys[i].Value, b.Keys[i].Value) {
return false
}
}
return true
})
// DefragmentReducer merges the current and next Key slices, returning a new Key
// slice.
//
// Implementations should modify and return `cur` to save on allocations, or
// consider allocating a new slice, as the `cur` slice may be retained by the
// DefragmentingIter and mutated. The `next` slice must not be mutated.
//
// The incoming slices are sorted by (SeqNum, Kind) descending. The output slice
// must also have this sort order.
type DefragmentReducer func(cur, next []Key) []Key
// StaticDefragmentReducer is a no-op DefragmentReducer that simply returns the
// current key slice, effectively retaining the first set of keys encountered
// for a defragmented span.
//
// This reducer can be used, for example, when the set of Keys for each Span
// being reduced is not expected to change, and therefore the keys from the
// first span encountered can be used without considering keys in subsequent
// spans.
var StaticDefragmentReducer DefragmentReducer = func(cur, _ []Key) []Key {
return cur
}
// iterPos is an enum indicating the position of the defragmenting iter's
// wrapped iter. The defragmenting iter must look ahead or behind when
// defragmenting forward or backwards respectively, and this enum records that
// current position.
type iterPos int8
const (
iterPosPrev iterPos = -1
iterPosCurr iterPos = 0
iterPosNext iterPos = +1
)
// DefragmentingIter wraps a key span iterator, defragmenting physical
// fragmentation during iteration.
//
// During flushes and compactions, keys applied over a span may be split at
// sstable boundaries. This fragmentation can produce internal key bounds that
// do not match any of the bounds ever supplied to a user operation. This
// physical fragmentation is necessary to avoid excessively wide sstables.
//
// The defragmenting iterator undoes this physical fragmentation, joining spans
// with abutting bounds and equal state. The defragmenting iterator takes a
// DefragmentMethod to determine what is "equal state" for a span. The
// DefragmentMethod is a function type, allowing arbitrary comparisons between
// Span keys.
//
// Seeking (SeekGE, SeekLT) poses an obstacle to defragmentation. A seek may
// land on a physical fragment in the middle of several fragments that must be
// defragmented. A seek that lands in a fragment straddling the seek key must
// first degfragment in the opposite direction of iteration to find the
// beginning of the defragmented span, and then defragments in the iteration
// direction, ensuring it's found a whole defragmented span.
type DefragmentingIter struct {
// DefragmentingBuffers holds buffers used for copying iterator state.
*DefragmentingBuffers
comparer *base.Comparer
equal base.Equal
iter FragmentIterator
iterSpan *Span
iterPos iterPos
// curr holds the span at the current iterator position.
curr Span
// method is a comparison function for two spans. method is called when two
// spans are abutting to determine whether they may be defragmented.
// method does not itself check for adjacency for the two spans.
method DefragmentMethod
// reduce is the reducer function used to collect Keys across all spans that
// constitute a defragmented span.
reduce DefragmentReducer
}
// DefragmentingBuffers holds buffers used for copying iterator state.
type DefragmentingBuffers struct {
// currBuf is a buffer for use when copying user keys for curr. currBuf is
// cleared between positioning methods.
currBuf bytealloc.A
// keysBuf is a buffer for use when copying Keys for DefragmentingIter.curr.
keysBuf []Key
// keyBuf is a buffer specifically for the defragmented start key when
// defragmenting backwards or the defragmented end key when defragmenting
// forwards. These bounds are overwritten repeatedly during defragmentation,
// and the defragmentation routines overwrite keyBuf repeatedly to store
// these extended bounds.
keyBuf []byte
}
// PrepareForReuse discards any excessively large buffers.
func (bufs *DefragmentingBuffers) PrepareForReuse() {
if cap(bufs.currBuf) > bufferReuseMaxCapacity {
bufs.currBuf = nil
}
if cap(bufs.keyBuf) > bufferReuseMaxCapacity {
bufs.keyBuf = nil
}
if cap(bufs.keysBuf) > keysReuseMaxCapacity {
bufs.keysBuf = nil
}
}
// Assert that *DefragmentingIter implements the FragmentIterator interface.
var _ FragmentIterator = (*DefragmentingIter)(nil)
// Init initializes the defragmenting iter using the provided defragment
// method.
func (i *DefragmentingIter) Init(
comparer *base.Comparer,
iter FragmentIterator,
equal DefragmentMethod,
reducer DefragmentReducer,
bufs *DefragmentingBuffers,
) {
*i = DefragmentingIter{
DefragmentingBuffers: bufs,
comparer: comparer,
equal: comparer.Equal,
iter: iter,
method: equal,
reduce: reducer,
}
}
// Error returns any accumulated error.
func (i *DefragmentingIter) Error() error {
return i.iter.Error()
}
// Close closes the underlying iterators.
func (i *DefragmentingIter) Close() error {
return i.iter.Close()
}
// SeekGE moves the iterator to the first span covering a key greater than or
// equal to the given key. This is equivalent to seeking to the first span with
// an end key greater than the given key.
func (i *DefragmentingIter) SeekGE(key []byte) *Span {
i.iterSpan = i.iter.SeekGE(key)
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
} else if i.iterSpan.Empty() {
i.iterPos = iterPosCurr
return i.iterSpan
}
// If the span starts strictly after key, we know there mustn't be an
// earlier span that ends at i.iterSpan.Start, otherwise i.iter would've
// returned that span instead.
if i.comparer.Compare(i.iterSpan.Start, key) > 0 {
return i.defragmentForward()
}
// The span we landed on has a Start bound ≤ key. There may be additional
// fragments before this span. Defragment backward to find the start of the
// defragmented span.
i.defragmentBackward()
if i.iterPos == iterPosPrev {
// Next once back onto the span.
i.iterSpan = i.iter.Next()
}
// Defragment the full span from its start.
return i.defragmentForward()
}
// SeekLT moves the iterator to the last span covering a key less than the
// given key. This is equivalent to seeking to the last span with a start
// key less than the given key.
func (i *DefragmentingIter) SeekLT(key []byte) *Span {
i.iterSpan = i.iter.SeekLT(key)
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
} else if i.iterSpan.Empty() {
i.iterPos = iterPosCurr
return i.iterSpan
}
// If the span ends strictly before key, we know there mustn't be a later
// span that starts at i.iterSpan.End, otherwise i.iter would've returned
// that span instead.
if i.comparer.Compare(i.iterSpan.End, key) < 0 {
return i.defragmentBackward()
}
// The span we landed on has a End bound ≥ key. There may be additional
// fragments after this span. Defragment forward to find the end of the
// defragmented span.
i.defragmentForward()
if i.iterPos == iterPosNext {
// Prev once back onto the span.
i.iterSpan = i.iter.Prev()
}
// Defragment the full span from its end.
return i.defragmentBackward()
}
// First seeks the iterator to the first span and returns it.
func (i *DefragmentingIter) First() *Span {
i.iterSpan = i.iter.First()
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
}
return i.defragmentForward()
}
// Last seeks the iterator to the last span and returns it.
func (i *DefragmentingIter) Last() *Span {
i.iterSpan = i.iter.Last()
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
}
return i.defragmentBackward()
}
// Next advances to the next span and returns it.
func (i *DefragmentingIter) Next() *Span {
switch i.iterPos {
case iterPosPrev:
// Switching directions; The iterator is currently positioned over the
// last span of the previous set of fragments. In the below diagram,
// the iterator is positioned over the last span that contributes to
// the defragmented x position. We want to be positioned over the first
// span that contributes to the z position.
//
// x x x y y y z z z
// ^ ^
// old new
//
// Next once to move onto y, defragment forward to land on the first z
// position.
i.iterSpan = i.iter.Next()
if invariants.Enabled && i.iterSpan == nil {
panic("pebble: invariant violation: no next span while switching directions")
}
// We're now positioned on the first span that was defragmented into the
// current iterator position. Skip over the rest of the current iterator
// position's constitutent fragments. In the above example, this would
// land on the first 'z'.
i.defragmentForward()
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
}
// Now that we're positioned over the first of the next set of
// fragments, defragment forward.
return i.defragmentForward()
case iterPosCurr:
// iterPosCurr is only used when the iter is exhausted or when the iterator
// is at an empty span.
if invariants.Enabled && i.iterSpan != nil && !i.iterSpan.Empty() {
panic("pebble: invariant violation: iterPosCurr with valid iterSpan")
}
i.iterSpan = i.iter.Next()
if i.iterSpan == nil {
return nil
}
return i.defragmentForward()
case iterPosNext:
// Already at the next span.
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
}
return i.defragmentForward()
default:
panic("unreachable")
}
}
// Prev steps back to the previous span and returns it.
func (i *DefragmentingIter) Prev() *Span {
switch i.iterPos {
case iterPosPrev:
// Already at the previous span.
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
}
return i.defragmentBackward()
case iterPosCurr:
// iterPosCurr is only used when the iter is exhausted or when the iterator
// is at an empty span.
if invariants.Enabled && i.iterSpan != nil && !i.iterSpan.Empty() {
panic("pebble: invariant violation: iterPosCurr with valid iterSpan")
}
i.iterSpan = i.iter.Prev()
if i.iterSpan == nil {
return nil
}
return i.defragmentBackward()
case iterPosNext:
// Switching directions; The iterator is currently positioned over the
// first fragment of the next set of fragments. In the below diagram,
// the iterator is positioned over the first span that contributes to
// the defragmented z position. We want to be positioned over the last
// span that contributes to the x position.
//
// x x x y y y z z z
// ^ ^
// new old
//
// Prev once to move onto y, defragment backward to land on the last x
// position.
i.iterSpan = i.iter.Prev()
if invariants.Enabled && i.iterSpan == nil {
panic("pebble: invariant violation: no previous span while switching directions")
}
// We're now positioned on the last span that was defragmented into the
// current iterator position. Skip over the rest of the current iterator
// position's constitutent fragments. In the above example, this would
// land on the last 'x'.
i.defragmentBackward()
// Now that we're positioned over the last of the prev set of
// fragments, defragment backward.
if i.iterSpan == nil {
i.iterPos = iterPosCurr
return nil
}
return i.defragmentBackward()
default:
panic("unreachable")
}
}
// checkEqual checks the two spans for logical equivalence. It uses the passed-in
// DefragmentMethod and ensures both spans are NOT empty; not defragmenting empty
// spans is an optimization that lets us load fewer sstable blocks.
func (i *DefragmentingIter) checkEqual(left, right *Span) bool {
return (!left.Empty() && !right.Empty()) && i.method.ShouldDefragment(i.equal, i.iterSpan, &i.curr)
}
// defragmentForward defragments spans in the forward direction, starting from
// i.iter's current position. The span at the current position must be non-nil,
// but may be Empty().
func (i *DefragmentingIter) defragmentForward() *Span {
if i.iterSpan.Empty() {
// An empty span will never be equal to another span; see checkEqual for
// why. To avoid loading non-empty range keys further ahead by calling Next,
// return early.
i.iterPos = iterPosCurr
return i.iterSpan
}
i.saveCurrent()
i.iterPos = iterPosNext
i.iterSpan = i.iter.Next()
for i.iterSpan != nil {
if !i.equal(i.curr.End, i.iterSpan.Start) {
// Not a continuation.
break
}
if !i.checkEqual(i.iterSpan, &i.curr) {
// Not a continuation.
break
}
i.keyBuf = append(i.keyBuf[:0], i.iterSpan.End...)
i.curr.End = i.keyBuf
i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
i.iterSpan = i.iter.Next()
}
i.curr.Keys = i.keysBuf
return &i.curr
}
// defragmentBackward defragments spans in the backward direction, starting from
// i.iter's current position. The span at the current position must be non-nil,
// but may be Empty().
func (i *DefragmentingIter) defragmentBackward() *Span {
if i.iterSpan.Empty() {
// An empty span will never be equal to another span; see checkEqual for
// why. To avoid loading non-empty range keys further ahead by calling Next,
// return early.
i.iterPos = iterPosCurr
return i.iterSpan
}
i.saveCurrent()
i.iterPos = iterPosPrev
i.iterSpan = i.iter.Prev()
for i.iterSpan != nil {
if !i.equal(i.curr.Start, i.iterSpan.End) {
// Not a continuation.
break
}
if !i.checkEqual(i.iterSpan, &i.curr) {
// Not a continuation.
break
}
i.keyBuf = append(i.keyBuf[:0], i.iterSpan.Start...)
i.curr.Start = i.keyBuf
i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
i.iterSpan = i.iter.Prev()
}
i.curr.Keys = i.keysBuf
return &i.curr
}
func (i *DefragmentingIter) saveCurrent() {
i.currBuf.Reset()
i.keysBuf = i.keysBuf[:0]
i.keyBuf = i.keyBuf[:0]
if i.iterSpan == nil {
return
}
i.curr = Span{
Start: i.saveBytes(i.iterSpan.Start),
End: i.saveBytes(i.iterSpan.End),
KeysOrder: i.iterSpan.KeysOrder,
}
for j := range i.iterSpan.Keys {
i.keysBuf = append(i.keysBuf, Key{
Trailer: i.iterSpan.Keys[j].Trailer,
Suffix: i.saveBytes(i.iterSpan.Keys[j].Suffix),
Value: i.saveBytes(i.iterSpan.Keys[j].Value),
})
}
i.curr.Keys = i.keysBuf
}
func (i *DefragmentingIter) saveBytes(b []byte) []byte {
if b == nil {
return nil
}
i.currBuf, b = i.currBuf.Copy(b)
return b
}

13
vendor/github.com/cockroachdb/pebble/internal/keyspan/doc.go generated vendored Normal file
View File

@@ -0,0 +1,13 @@
// Package keyspan provides facilities for sorting, fragmenting and
// iterating over spans of user keys.
//
// A Span represents a range of user key space with an inclusive start
// key and exclusive end key. A span may hold any number of Keys which are
// applied over the entirety of the span's keyspace.
//
// Spans are used within Pebble as an in-memory representation of range
// deletion tombstones, and range key sets, unsets and deletes. Spans
// are fragmented at overlapping key boundaries by the Fragmenter type.
// This package's various iteration facilities require these
// non-overlapping fragmented spans.
package keyspan

98
vendor/github.com/cockroachdb/pebble/internal/keyspan/filter.go generated vendored Normal file
View File

@@ -0,0 +1,98 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
// FilterFunc defines a transform from the input Span into the output Span. The
// function returns true if the Span should be returned by the iterator, and
// false if the Span should be skipped. The FilterFunc is permitted to mutate
// the output Span, for example, to elice certain keys, or update the Span's
// bounds if so desired. The output Span's Keys slice may be reused to reduce
// allocations.
type FilterFunc func(in *Span, out *Span) (keep bool)
// filteringIter is a FragmentIterator that uses a FilterFunc to select which
// Spans from the input iterator are returned in the output.
//
// A note on Span lifetimes: as the FilterFunc reuses a Span with a mutable
// slice of Keys to reduce allocations, Spans returned by this iterator are only
// valid until the next relative or absolute positioning method is called.
type filteringIter struct {
iter FragmentIterator
filterFn FilterFunc
// span is a mutable Span passed to the filterFn. The filterFn is free to
// mutate this Span. The slice of Keys in the Span is reused with every call
// to the filterFn.
span Span
}
var _ FragmentIterator = (*filteringIter)(nil)
// Filter returns a new filteringIter that will filter the Spans from the
// provided child iterator using the provided FilterFunc.
func Filter(iter FragmentIterator, filter FilterFunc) FragmentIterator {
return &filteringIter{iter: iter, filterFn: filter}
}
// SeekGE implements FragmentIterator.
func (i *filteringIter) SeekGE(key []byte) *Span {
return i.filter(i.iter.SeekGE(key), +1)
}
// SeekLT implements FragmentIterator.
func (i *filteringIter) SeekLT(key []byte) *Span {
return i.filter(i.iter.SeekLT(key), -1)
}
// First implements FragmentIterator.
func (i *filteringIter) First() *Span {
return i.filter(i.iter.First(), +1)
}
// Last implements FragmentIterator.
func (i *filteringIter) Last() *Span {
return i.filter(i.iter.Last(), -1)
}
// Next implements FragmentIterator.
func (i *filteringIter) Next() *Span {
return i.filter(i.iter.Next(), +1)
}
// Prev implements FragmentIterator.
func (i *filteringIter) Prev() *Span {
return i.filter(i.iter.Prev(), -1)
}
// Error implements FragmentIterator.
func (i *filteringIter) Error() error {
return i.iter.Error()
}
// Close implements FragmentIterator.
func (i *filteringIter) Close() error {
return i.iter.Close()
}
// filter uses the filterFn (if configured) to filter and possibly mutate the
// given Span. If the current Span is to be skipped, the iterator continues
// iterating in the given direction until it lands on a Span that should be
// returned, or the iterator becomes invalid.
func (i *filteringIter) filter(span *Span, dir int8) *Span {
if i.filterFn == nil {
return span
}
for i.Error() == nil && span != nil {
if keep := i.filterFn(span, &i.span); keep {
return &i.span
}
if dir == +1 {
span = i.iter.Next()
} else {
span = i.iter.Prev()
}
}
return span
}

483
vendor/github.com/cockroachdb/pebble/internal/keyspan/fragmenter.go generated vendored Normal file
View File

@@ -0,0 +1,483 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"fmt"
"sort"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
type spansByStartKey struct {
cmp base.Compare
buf []Span
}
func (v *spansByStartKey) Len() int { return len(v.buf) }
func (v *spansByStartKey) Less(i, j int) bool {
return v.cmp(v.buf[i].Start, v.buf[j].Start) < 0
}
func (v *spansByStartKey) Swap(i, j int) {
v.buf[i], v.buf[j] = v.buf[j], v.buf[i]
}
type spansByEndKey struct {
cmp base.Compare
buf []Span
}
func (v *spansByEndKey) Len() int { return len(v.buf) }
func (v *spansByEndKey) Less(i, j int) bool {
return v.cmp(v.buf[i].End, v.buf[j].End) < 0
}
func (v *spansByEndKey) Swap(i, j int) {
v.buf[i], v.buf[j] = v.buf[j], v.buf[i]
}
// keysBySeqNumKind sorts spans by the start key's sequence number in
// descending order. If two spans have equal sequence number, they're compared
// by key kind in descending order. This ordering matches the ordering of
// base.InternalCompare among keys with matching user keys.
type keysBySeqNumKind []Key
func (v *keysBySeqNumKind) Len() int { return len(*v) }
func (v *keysBySeqNumKind) Less(i, j int) bool { return (*v)[i].Trailer > (*v)[j].Trailer }
func (v *keysBySeqNumKind) Swap(i, j int) { (*v)[i], (*v)[j] = (*v)[j], (*v)[i] }
// Sort the spans by start key. This is the ordering required by the
// Fragmenter. Usually spans are naturally sorted by their start key,
// but that isn't true for range deletion tombstones in the legacy
// range-del-v1 block format.
func Sort(cmp base.Compare, spans []Span) {
sorter := spansByStartKey{
cmp: cmp,
buf: spans,
}
sort.Sort(&sorter)
}
// Fragmenter fragments a set of spans such that overlapping spans are
// split at their overlap points. The fragmented spans are output to the
// supplied Output function.
type Fragmenter struct {
Cmp base.Compare
Format base.FormatKey
// Emit is called to emit a fragmented span and its keys. Every key defined
// within the emitted Span applies to the entirety of the Span's key span.
// Keys are ordered in decreasing order of their sequence numbers, and if
// equal, decreasing order of key kind.
Emit func(Span)
// pending contains the list of pending fragments that have not been
// flushed to the block writer. Note that the spans have not been
// fragmented on the end keys yet. That happens as the spans are
// flushed. All pending spans have the same Start.
pending []Span
// doneBuf is used to buffer completed span fragments when flushing to a
// specific key (e.g. TruncateAndFlushTo). It is cached in the Fragmenter to
// allow reuse.
doneBuf []Span
// sortBuf is used to sort fragments by end key when flushing.
sortBuf spansByEndKey
// flushBuf is used to sort keys by (seqnum,kind) before emitting.
flushBuf keysBySeqNumKind
// flushedKey is the key that fragments have been flushed up to. Any
// additional spans added to the fragmenter must have a start key >=
// flushedKey. A nil value indicates flushedKey has not been set.
flushedKey []byte
finished bool
}
func (f *Fragmenter) checkInvariants(buf []Span) {
for i := 1; i < len(buf); i++ {
if f.Cmp(buf[i].Start, buf[i].End) >= 0 {
panic(fmt.Sprintf("pebble: empty pending span invariant violated: %s", buf[i]))
}
if f.Cmp(buf[i-1].Start, buf[i].Start) != 0 {
panic(fmt.Sprintf("pebble: pending span invariant violated: %s %s",
f.Format(buf[i-1].Start), f.Format(buf[i].Start)))
}
}
}
// Add adds a span to the fragmenter. Spans may overlap and the
// fragmenter will internally split them. The spans must be presented in
// increasing start key order. That is, Add must be called with a series
// of spans like:
//
// a---e
// c---g
// c-----i
// j---n
// j-l
//
// We need to fragment the spans at overlap points. In the above
// example, we'd create:
//
// a-c-e
// c-e-g
// c-e-g-i
// j-l-n
// j-l
//
// The fragments need to be output sorted by start key, and for equal start
// keys, sorted by descending sequence number. This last part requires a mild
// bit of care as the fragments are not created in descending sequence number
// order.
//
// Once a start key has been seen, we know that we'll never see a smaller
// start key and can thus flush all of the fragments that lie before that
// start key.
//
// Walking through the example above, we start with:
//
// a---e
//
// Next we add [c,g) resulting in:
//
// a-c-e
// c---g
//
// The fragment [a,c) is flushed leaving the pending spans as:
//
// c-e
// c---g
//
// The next span is [c,i):
//
// c-e
// c---g
// c-----i
//
// No fragments are flushed. The next span is [j,n):
//
// c-e
// c---g
// c-----i
// j---n
//
// The fragments [c,e), [c,g) and [c,i) are flushed. We sort these fragments
// by their end key, then split the fragments on the end keys:
//
// c-e
// c-e-g
// c-e---i
//
// The [c,e) fragments all get flushed leaving:
//
// e-g
// e---i
//
// This process continues until there are no more fragments to flush.
//
// WARNING: the slices backing Start, End, Keys, Key.Suffix and Key.Value are
// all retained after this method returns and should not be modified. This is
// safe for spans that are added from a memtable or batch. It is partially
// unsafe for a span read from an sstable. Specifically, the Keys slice of a
// Span returned during sstable iteration is only valid until the next iterator
// operation. The stability of the user keys depend on whether the block is
// prefix compressed, and in practice Pebble never prefix compresses range
// deletion and range key blocks, so these keys are stable. Because of this key
// stability, typically callers only need to perform a shallow clone of the Span
// before Add-ing it to the fragmenter.
//
// Add requires the provided span's keys are sorted in Trailer descending order.
func (f *Fragmenter) Add(s Span) {
if f.finished {
panic("pebble: span fragmenter already finished")
} else if s.KeysOrder != ByTrailerDesc {
panic("pebble: span keys unexpectedly not in trailer descending order")
}
if f.flushedKey != nil {
switch c := f.Cmp(s.Start, f.flushedKey); {
case c < 0:
panic(fmt.Sprintf("pebble: start key (%s) < flushed key (%s)",
f.Format(s.Start), f.Format(f.flushedKey)))
}
}
if f.Cmp(s.Start, s.End) >= 0 {
// An empty span, we can ignore it.
return
}
if invariants.RaceEnabled {
f.checkInvariants(f.pending)
defer func() { f.checkInvariants(f.pending) }()
}
if len(f.pending) > 0 {
// Since all of the pending spans have the same start key, we only need
// to compare against the first one.
switch c := f.Cmp(f.pending[0].Start, s.Start); {
case c > 0:
panic(fmt.Sprintf("pebble: keys must be added in order: %s > %s",
f.Format(f.pending[0].Start), f.Format(s.Start)))
case c == 0:
// The new span has the same start key as the existing pending
// spans. Add it to the pending buffer.
f.pending = append(f.pending, s)
return
}
// At this point we know that the new start key is greater than the pending
// spans start keys.
f.truncateAndFlush(s.Start)
}
f.pending = append(f.pending, s)
}
// Cover is returned by Framenter.Covers and describes a span's relationship to
// a key at a particular snapshot.
type Cover int8
const (
// NoCover indicates the tested key does not fall within the span's bounds,
// or the span contains no keys with sequence numbers higher than the key's.
NoCover Cover = iota
// CoversInvisibly indicates the tested key does fall within the span's
// bounds and the span contains at least one key with a higher sequence
// number, but none visible at the provided snapshot.
CoversInvisibly
// CoversVisibly indicates the tested key does fall within the span's
// bounds, and the span constains at least one key with a sequence number
// higher than the key's sequence number that is visible at the provided
// snapshot.
CoversVisibly
)
// Covers returns an enum indicating whether the specified key is covered by one
// of the pending keys. The provided key must be consistent with the ordering of
// the spans. That is, it is invalid to specify a key here that is out of order
// with the span start keys passed to Add.
func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) Cover {
if f.finished {
panic("pebble: span fragmenter already finished")
}
if len(f.pending) == 0 {
return NoCover
}
if f.Cmp(f.pending[0].Start, key.UserKey) > 0 {
panic(fmt.Sprintf("pebble: keys must be in order: %s > %s",
f.Format(f.pending[0].Start), key.Pretty(f.Format)))
}
cover := NoCover
seqNum := key.SeqNum()
for _, s := range f.pending {
if f.Cmp(key.UserKey, s.End) < 0 {
// NB: A range deletion tombstone does not delete a point operation
// at the same sequence number, and broadly a span is not considered
// to cover a point operation at the same sequence number.
for i := range s.Keys {
if kseq := s.Keys[i].SeqNum(); kseq > seqNum {
// This key from the span has a higher sequence number than
// `key`. It covers `key`, although the span's key might not
// be visible if its snapshot is too high.
//
// Batch keys are always be visible.
if kseq < snapshot || kseq&base.InternalKeySeqNumBatch != 0 {
return CoversVisibly
}
// s.Keys[i] is not visible.
cover = CoversInvisibly
}
}
}
}
return cover
}
// Empty returns true if all fragments added so far have finished flushing.
func (f *Fragmenter) Empty() bool {
return f.finished || len(f.pending) == 0
}
// TruncateAndFlushTo flushes all of the fragments with a start key <= key,
// truncating spans to the specified end key. Used during compaction to force
// emitting of spans which straddle an sstable boundary. Consider
// the scenario:
//
// a---------k#10
// f#8
// f#7
//
// Let's say the next user key after f is g. Calling TruncateAndFlushTo(g) will
// flush this span:
//
// a-------g#10
// f#8
// f#7
//
// And leave this one in f.pending:
//
// g----k#10
//
// WARNING: The fragmenter could hold on to the specified end key. Ensure it's
// a safe byte slice that could outlast the current sstable output, and one
// that will never be modified.
func (f *Fragmenter) TruncateAndFlushTo(key []byte) {
if f.finished {
panic("pebble: span fragmenter already finished")
}
if f.flushedKey != nil {
switch c := f.Cmp(key, f.flushedKey); {
case c < 0:
panic(fmt.Sprintf("pebble: start key (%s) < flushed key (%s)",
f.Format(key), f.Format(f.flushedKey)))
}
}
if invariants.RaceEnabled {
f.checkInvariants(f.pending)
defer func() { f.checkInvariants(f.pending) }()
}
if len(f.pending) > 0 {
// Since all of the pending spans have the same start key, we only need
// to compare against the first one.
switch c := f.Cmp(f.pending[0].Start, key); {
case c > 0:
panic(fmt.Sprintf("pebble: keys must be added in order: %s > %s",
f.Format(f.pending[0].Start), f.Format(key)))
case c == 0:
return
}
}
f.truncateAndFlush(key)
}
// Start returns the start key of the first span in the pending buffer, or nil
// if there are no pending spans. The start key of all pending spans is the same
// as that of the first one.
func (f *Fragmenter) Start() []byte {
if len(f.pending) > 0 {
return f.pending[0].Start
}
return nil
}
// Flushes all pending spans up to key (exclusive).
//
// WARNING: The specified key is stored without making a copy, so all callers
// must ensure it is safe.
func (f *Fragmenter) truncateAndFlush(key []byte) {
f.flushedKey = append(f.flushedKey[:0], key...)
done := f.doneBuf[:0]
pending := f.pending
f.pending = f.pending[:0]
// pending and f.pending share the same underlying storage. As we iterate
// over pending we append to f.pending, but only one entry is appended in
// each iteration, after we have read the entry being overwritten.
for _, s := range pending {
if f.Cmp(key, s.End) < 0 {
// s: a--+--e
// new: c------
if f.Cmp(s.Start, key) < 0 {
done = append(done, Span{
Start: s.Start,
End: key,
Keys: s.Keys,
})
}
f.pending = append(f.pending, Span{
Start: key,
End: s.End,
Keys: s.Keys,
})
} else {
// s: a-----e
// new: e----
done = append(done, s)
}
}
f.doneBuf = done[:0]
f.flush(done, nil)
}
// flush a group of range spans to the block. The spans are required to all have
// the same start key. We flush all span fragments until startKey > lastKey. If
// lastKey is nil, all span fragments are flushed. The specification of a
// non-nil lastKey occurs for range deletion tombstones during compaction where
// we want to flush (but not truncate) all range tombstones that start at or
// before the first key in the next sstable. Consider:
//
// a---e#10
// a------h#9
//
// If a compaction splits the sstables at key c we want the first sstable to
// contain the tombstones [a,e)#10 and [a,e)#9. Fragmentation would naturally
// produce a tombstone [e,h)#9, but we don't need to output that tombstone to
// the first sstable.
func (f *Fragmenter) flush(buf []Span, lastKey []byte) {
if invariants.RaceEnabled {
f.checkInvariants(buf)
}
// Sort the spans by end key. This will allow us to walk over the spans and
// easily determine the next split point (the smallest end-key).
f.sortBuf.cmp = f.Cmp
f.sortBuf.buf = buf
sort.Sort(&f.sortBuf)
// Loop over the spans, splitting by end key.
for len(buf) > 0 {
// A prefix of spans will end at split. remove represents the count of
// that prefix.
remove := 1
split := buf[0].End
f.flushBuf = append(f.flushBuf[:0], buf[0].Keys...)
for i := 1; i < len(buf); i++ {
if f.Cmp(split, buf[i].End) == 0 {
remove++
}
f.flushBuf = append(f.flushBuf, buf[i].Keys...)
}
sort.Sort(&f.flushBuf)
f.Emit(Span{
Start: buf[0].Start,
End: split,
// Copy the sorted keys to a new slice.
//
// This allocation is an unfortunate side effect of the Fragmenter and
// the expectation that the spans it produces are available in-memory
// indefinitely.
//
// Eventually, we should be able to replace the fragmenter with the
// keyspan.MergingIter which will perform just-in-time
// fragmentation, and only guaranteeing the memory lifetime for the
// current span. The MergingIter fragments while only needing to
// access one Span per level. It only accesses the Span at the
// current position for each level. During compactions, we can write
// these spans to sstables without retaining previous Spans.
Keys: append([]Key(nil), f.flushBuf...),
})
if lastKey != nil && f.Cmp(split, lastKey) > 0 {
break
}
// Adjust the start key for every remaining span.
buf = buf[remove:]
for i := range buf {
buf[i].Start = split
}
}
}
// Finish flushes any remaining fragments to the output. It is an error to call
// this if any other spans will be added.
func (f *Fragmenter) Finish() {
if f.finished {
panic("pebble: span fragmenter already finished")
}
f.flush(f.pending, nil)
f.finished = true
}

53
vendor/github.com/cockroachdb/pebble/internal/keyspan/get.go generated vendored Normal file
View File

@@ -0,0 +1,53 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// Get returns the newest span that contains the target key. If no span
// contains the target key, an empty span is returned. The snapshot
// parameter controls the visibility of spans (only spans older than the
// snapshot sequence number are visible). The iterator must contain
// fragmented spans: no span may overlap another.
func Get(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
// NB: We use SeekLT in order to land on the proper span for a search
// key that resides in the middle of a span. Consider the scenario:
//
// a---e
// e---i
//
// The spans are indexed by their start keys `a` and `e`. If the
// search key is `c` we want to land on the span [a,e). If we were
// to use SeekGE then the search key `c` would land on the span
// [e,i) and we'd have to backtrack. The one complexity here is what
// happens for the search key `e`. In that case SeekLT will land us
// on the span [a,e) and we'll have to move forward.
iterSpan := iter.SeekLT(key)
if iterSpan == nil {
iterSpan = iter.Next()
if iterSpan == nil {
// The iterator is empty.
return nil
}
if cmp(key, iterSpan.Start) < 0 {
// The search key lies before the first span.
return nil
}
}
// Invariant: key > iterSpan.Start
if cmp(key, iterSpan.End) >= 0 {
// The current span lies before the search key. Advance the iterator
// once to potentially land on a key with a start key exactly equal to
// key. (See the comment at the beginning of this function.)
iterSpan = iter.Next()
if iterSpan == nil || cmp(key, iterSpan.Start) < 0 {
// We've run out of spans or we've moved on to a span which
// starts after our search key.
return nil
}
}
return iterSpan
}

1046
vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

118
vendor/github.com/cockroachdb/pebble/internal/keyspan/internal_iter_shim.go generated vendored Normal file
View File

@@ -0,0 +1,118 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// InternalIteratorShim is a temporary iterator type used as a shim between
// keyspan.MergingIter and base.InternalIterator. It's used temporarily for
// range deletions during compactions, allowing range deletions to be
// interleaved by a compaction input iterator.
//
// TODO(jackson): This type should be removed, and the usages converted to using
// an InterleavingIterator type that interleaves keyspan.Spans from a
// keyspan.FragmentIterator with point keys.
type InternalIteratorShim struct {
miter MergingIter
mbufs MergingBuffers
span *Span
iterKey base.InternalKey
}
// Assert that InternalIteratorShim implements InternalIterator.
var _ base.InternalIterator = &InternalIteratorShim{}
// Init initializes the internal iterator shim to merge the provided fragment
// iterators.
func (i *InternalIteratorShim) Init(cmp base.Compare, iters ...FragmentIterator) {
i.miter.Init(cmp, noopTransform, &i.mbufs, iters...)
}
// Span returns the span containing the full set of keys over the key span at
// the current iterator position.
func (i *InternalIteratorShim) Span() *Span {
return i.span
}
// SeekGE implements (base.InternalIterator).SeekGE.
func (i *InternalIteratorShim) SeekGE(
key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// SeekPrefixGE implements (base.InternalIterator).SeekPrefixGE.
func (i *InternalIteratorShim) SeekPrefixGE(
prefix, key []byte, flags base.SeekGEFlags,
) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// SeekLT implements (base.InternalIterator).SeekLT.
func (i *InternalIteratorShim) SeekLT(
key []byte, flags base.SeekLTFlags,
) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// First implements (base.InternalIterator).First.
func (i *InternalIteratorShim) First() (*base.InternalKey, base.LazyValue) {
i.span = i.miter.First()
for i.span != nil && i.span.Empty() {
i.span = i.miter.Next()
}
if i.span == nil {
return nil, base.LazyValue{}
}
i.iterKey = base.InternalKey{UserKey: i.span.Start, Trailer: i.span.Keys[0].Trailer}
return &i.iterKey, base.MakeInPlaceValue(i.span.End)
}
// Last implements (base.InternalIterator).Last.
func (i *InternalIteratorShim) Last() (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// Next implements (base.InternalIterator).Next.
func (i *InternalIteratorShim) Next() (*base.InternalKey, base.LazyValue) {
i.span = i.miter.Next()
for i.span != nil && i.span.Empty() {
i.span = i.miter.Next()
}
if i.span == nil {
return nil, base.LazyValue{}
}
i.iterKey = base.InternalKey{UserKey: i.span.Start, Trailer: i.span.Keys[0].Trailer}
return &i.iterKey, base.MakeInPlaceValue(i.span.End)
}
// NextPrefix implements (base.InternalIterator).NextPrefix.
func (i *InternalIteratorShim) NextPrefix([]byte) (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// Prev implements (base.InternalIterator).Prev.
func (i *InternalIteratorShim) Prev() (*base.InternalKey, base.LazyValue) {
panic("unimplemented")
}
// Error implements (base.InternalIterator).Error.
func (i *InternalIteratorShim) Error() error {
return i.miter.Error()
}
// Close implements (base.InternalIterator).Close.
func (i *InternalIteratorShim) Close() error {
return i.miter.Close()
}
// SetBounds implements (base.InternalIterator).SetBounds.
func (i *InternalIteratorShim) SetBounds(lower, upper []byte) {
}
// String implements fmt.Stringer.
func (i *InternalIteratorShim) String() string {
return i.miter.String()
}

218
vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go generated vendored Normal file
View File

@@ -0,0 +1,218 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/manifest"
)
// FragmentIterator defines an iterator interface over spans. The spans
// surfaced by a FragmentIterator must be non-overlapping. This is achieved by
// fragmenting spans at overlap points (see Fragmenter).
//
// A Span returned by a FragmentIterator is only valid until the next
// positioning method. Some implementations (eg, keyspan.Iter) may provide
// longer lifetimes but implementations need only guarantee stability until the
// next positioning method.
type FragmentIterator interface {
// SeekGE moves the iterator to the first span covering a key greater than
// or equal to the given key. This is equivalent to seeking to the first
// span with an end key greater than the given key.
SeekGE(key []byte) *Span
// SeekLT moves the iterator to the last span covering a key less than the
// given key. This is equivalent to seeking to the last span with a start
// key less than the given key.
SeekLT(key []byte) *Span
// First moves the iterator to the first span.
First() *Span
// Last moves the iterator to the last span.
Last() *Span
// Next moves the iterator to the next span.
//
// It is valid to call Next when the iterator is positioned before the first
// key/value pair due to either a prior call to SeekLT or Prev which
// returned an invalid span. It is not allowed to call Next when the
// previous call to SeekGE, SeekPrefixGE or Next returned an invalid span.
Next() *Span
// Prev moves the iterator to the previous span.
//
// It is valid to call Prev when the iterator is positioned after the last
// key/value pair due to either a prior call to SeekGE or Next which
// returned an invalid span. It is not allowed to call Prev when the
// previous call to SeekLT or Prev returned an invalid span.
Prev() *Span
// Error returns any accumulated error.
Error() error
// Close closes the iterator and returns any accumulated error. Exhausting
// the iterator is not considered to be an error. It is valid to call Close
// multiple times. Other methods should not be called after the iterator has
// been closed.
Close() error
}
// TableNewSpanIter creates a new iterator for range key spans for the given
// file.
type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions SpanIterOptions) (FragmentIterator, error)
// SpanIterOptions is a subset of IterOptions that are necessary to instantiate
// per-sstable span iterators.
type SpanIterOptions struct {
// RangeKeyFilters can be used to avoid scanning tables and blocks in tables
// when iterating over range keys.
RangeKeyFilters []base.BlockPropertyFilter
}
// Iter is an iterator over a set of fragmented spans.
type Iter struct {
cmp base.Compare
spans []Span
index int
}
// Iter implements the FragmentIterator interface.
var _ FragmentIterator = (*Iter)(nil)
// NewIter returns a new iterator over a set of fragmented spans.
func NewIter(cmp base.Compare, spans []Span) *Iter {
i := &Iter{}
i.Init(cmp, spans)
return i
}
// Count returns the number of spans contained by Iter.
func (i *Iter) Count() int {
return len(i.spans)
}
// Init initializes an Iter with the provided spans.
func (i *Iter) Init(cmp base.Compare, spans []Span) {
*i = Iter{
cmp: cmp,
spans: spans,
index: -1,
}
}
// SeekGE implements FragmentIterator.SeekGE.
func (i *Iter) SeekGE(key []byte) *Span {
// NB: manually inlined sort.Search is ~5% faster.
//
// Define f(j) = false iff the span i.spans[j] is strictly before `key`
// (equivalently, i.spans[j].End ≤ key.)
//
// Define f(-1) == false and f(n) == true.
// Invariant: f(index-1) == false, f(upper) == true.
i.index = 0
upper := len(i.spans)
for i.index < upper {
h := int(uint(i.index+upper) >> 1) // avoid overflow when computing h
// i.index ≤ h < upper
if i.cmp(key, i.spans[h].End) >= 0 {
i.index = h + 1 // preserves f(i-1) == false
} else {
upper = h // preserves f(j) == true
}
}
// i.index == upper, f(i.index-1) == false, and f(upper) (= f(i.index)) ==
// true => answer is i.index.
if i.index >= len(i.spans) {
return nil
}
return &i.spans[i.index]
}
// SeekLT implements FragmentIterator.SeekLT.
func (i *Iter) SeekLT(key []byte) *Span {
// NB: manually inlined sort.Search is ~5% faster.
//
// Define f(-1) == false and f(n) == true.
// Invariant: f(index-1) == false, f(upper) == true.
i.index = 0
upper := len(i.spans)
for i.index < upper {
h := int(uint(i.index+upper) >> 1) // avoid overflow when computing h
// i.index ≤ h < upper
if i.cmp(key, i.spans[h].Start) > 0 {
i.index = h + 1 // preserves f(i-1) == false
} else {
upper = h // preserves f(j) == true
}
}
// i.index == upper, f(i.index-1) == false, and f(upper) (= f(i.index)) ==
// true => answer is i.index.
// Since keys are strictly increasing, if i.index > 0 then i.index-1 will be
// the largest whose key is < the key sought.
i.index--
if i.index < 0 {
return nil
}
return &i.spans[i.index]
}
// First implements FragmentIterator.First.
func (i *Iter) First() *Span {
if len(i.spans) == 0 {
return nil
}
i.index = 0
return &i.spans[i.index]
}
// Last implements FragmentIterator.Last.
func (i *Iter) Last() *Span {
if len(i.spans) == 0 {
return nil
}
i.index = len(i.spans) - 1
return &i.spans[i.index]
}
// Next implements FragmentIterator.Next.
func (i *Iter) Next() *Span {
if i.index >= len(i.spans) {
return nil
}
i.index++
if i.index >= len(i.spans) {
return nil
}
return &i.spans[i.index]
}
// Prev implements FragmentIterator.Prev.
func (i *Iter) Prev() *Span {
if i.index < 0 {
return nil
}
i.index--
if i.index < 0 {
return nil
}
return &i.spans[i.index]
}
// Error implements FragmentIterator.Error.
func (i *Iter) Error() error {
return nil
}
// Close implements FragmentIterator.Close.
func (i *Iter) Close() error {
return nil
}
func (i *Iter) String() string {
return "fragmented-spans"
}

521
vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go generated vendored Normal file
View File

@@ -0,0 +1,521 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import (
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/manifest"
)
// LevelIter provides a merged view of spans from sstables in a level.
// It takes advantage of level invariants to only have one sstable span block
// open at one time, opened using the newIter function passed in.
type LevelIter struct {
cmp base.Compare
// Denotes the kind of key the level iterator should read. If the key type
// is KeyTypePoint, the level iterator will read range tombstones (which
// only affect point keys). If the key type is KeyTypeRange, the level
// iterator will read range keys. It is invalid to configure an iterator
// with the KeyTypePointAndRange key type.
//
// If key type is KeyTypePoint, no straddle spans are emitted between files,
// and point key bounds are used to find files instead of range key bounds.
//
// TODO(bilal): Straddle spans can safely be produced in rangedel mode once
// we can guarantee that we will never read sstables in a level that split
// user keys across them. This might be guaranteed in a future release, but
// as of CockroachDB 22.2 it is not guaranteed, so to be safe disable it when
// keyType == KeyTypePoint
keyType manifest.KeyType
// The LSM level this LevelIter is initialized for. Used in logging.
level manifest.Level
// The below fields are used to fill in gaps between adjacent files' range
// key spaces. This is an optimization to avoid unnecessarily loading files
// in cases where range keys are sparse and rare. dir is set by every
// positioning operation, straddleDir is set to dir whenever a straddling
// Span is synthesized and the last positioning operation returned a
// synthesized straddle span.
//
// Note that when a straddle span is initialized, iterFile is modified to
// point to the next file in the straddleDir direction. A change of direction
// on a straddle key therefore necessitates the value of iterFile to be
// reverted.
dir int
straddle Span
straddleDir int
// The iter for the current file (iterFile). It is nil under any of the
// following conditions:
// - files.Current() == nil
// - err != nil
// - straddleDir != 0, in which case iterFile is not nil and points to the
// next file (in the straddleDir direction).
// - some other constraint, like the bounds in opts, caused the file at index to not
// be relevant to the iteration.
iter FragmentIterator
// iterFile holds the current file.
// INVARIANT: iterFile = files.Current()
iterFile *manifest.FileMetadata
newIter TableNewSpanIter
files manifest.LevelIterator
err error
// The options that were passed in.
tableOpts SpanIterOptions
// TODO(bilal): Add InternalIteratorStats.
}
// LevelIter implements the keyspan.FragmentIterator interface.
var _ FragmentIterator = (*LevelIter)(nil)
// NewLevelIter returns a LevelIter.
func NewLevelIter(
opts SpanIterOptions,
cmp base.Compare,
newIter TableNewSpanIter,
files manifest.LevelIterator,
level manifest.Level,
keyType manifest.KeyType,
) *LevelIter {
l := &LevelIter{}
l.Init(opts, cmp, newIter, files, level, keyType)
return l
}
// Init initializes a LevelIter.
func (l *LevelIter) Init(
opts SpanIterOptions,
cmp base.Compare,
newIter TableNewSpanIter,
files manifest.LevelIterator,
level manifest.Level,
keyType manifest.KeyType,
) {
l.err = nil
l.level = level
l.tableOpts = opts
l.cmp = cmp
l.iterFile = nil
l.newIter = newIter
switch keyType {
case manifest.KeyTypePoint:
l.keyType = keyType
l.files = files.Filter(keyType)
case manifest.KeyTypeRange:
l.keyType = keyType
l.files = files.Filter(keyType)
default:
panic(fmt.Sprintf("unsupported key type: %v", keyType))
}
}
func (l *LevelIter) findFileGE(key []byte) *manifest.FileMetadata {
// Find the earliest file whose largest key is >= key.
//
// If the earliest file has its largest key == key and that largest key is a
// range deletion sentinel, we know that we manufactured this sentinel to convert
// the exclusive range deletion end key into an inclusive key (reminder: [start, end)#seqnum
// is the form of a range deletion sentinel which can contribute a largest key = end#sentinel).
// In this case we don't return this as the earliest file since there is nothing actually
// equal to key in it.
m := l.files.SeekGE(l.cmp, key)
for m != nil {
largestKey := m.LargestRangeKey
if l.keyType == manifest.KeyTypePoint {
largestKey = m.LargestPointKey
}
if !largestKey.IsExclusiveSentinel() || l.cmp(largestKey.UserKey, key) != 0 {
break
}
m = l.files.Next()
}
return m
}
func (l *LevelIter) findFileLT(key []byte) *manifest.FileMetadata {
// Find the last file whose smallest key is < key.
return l.files.SeekLT(l.cmp, key)
}
type loadFileReturnIndicator int8
const (
noFileLoaded loadFileReturnIndicator = iota
fileAlreadyLoaded
newFileLoaded
)
func (l *LevelIter) loadFile(file *manifest.FileMetadata, dir int) loadFileReturnIndicator {
indicator := noFileLoaded
if l.iterFile == file {
if l.err != nil {
return noFileLoaded
}
if l.iter != nil {
// We are already at the file, but we would need to check for bounds.
// Set indicator accordingly.
indicator = fileAlreadyLoaded
}
// We were already at file, but don't have an iterator, probably because the file was
// beyond the iteration bounds. It may still be, but it is also possible that the bounds
// have changed. We handle that below.
}
// Note that LevelIter.Close() can be called multiple times.
if indicator != fileAlreadyLoaded {
if err := l.Close(); err != nil {
return noFileLoaded
}
}
l.iterFile = file
if file == nil {
return noFileLoaded
}
if indicator != fileAlreadyLoaded {
l.iter, l.err = l.newIter(file, l.tableOpts)
indicator = newFileLoaded
}
if l.err != nil {
return noFileLoaded
}
return indicator
}
// SeekGE implements keyspan.FragmentIterator.
func (l *LevelIter) SeekGE(key []byte) *Span {
l.dir = +1
l.straddle = Span{}
l.straddleDir = 0
l.err = nil // clear cached iteration error
f := l.findFileGE(key)
if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(key, f.SmallestRangeKey.UserKey) < 0 {
// Peek at the previous file.
prevFile := l.files.Prev()
l.files.Next()
if prevFile != nil {
// We could unconditionally return an empty span between the seek key and
// f.SmallestRangeKey, however if this span is to the left of all range
// keys on this level, it could lead to inconsistent behaviour in relative
// positioning operations. Consider this example, with a b-c range key:
//
// SeekGE(a) -> a-b:{}
// Next() -> b-c{(#5,RANGEKEYSET,@4,foo)}
// Prev() -> nil
//
// Iterators higher up in the iterator stack rely on this sort of relative
// positioning consistency.
//
// TODO(bilal): Investigate ways to be able to return straddle spans in
// cases similar to the above, while still retaining correctness.
// Return a straddling key instead of loading the file.
l.iterFile = f
if err := l.Close(); err != nil {
return l.verify(nil)
}
l.straddleDir = +1
l.straddle = Span{
Start: prevFile.LargestRangeKey.UserKey,
End: f.SmallestRangeKey.UserKey,
Keys: nil,
}
return l.verify(&l.straddle)
}
}
loadFileIndicator := l.loadFile(f, +1)
if loadFileIndicator == noFileLoaded {
return l.verify(nil)
}
if span := l.iter.SeekGE(key); span != nil {
return l.verify(span)
}
return l.skipEmptyFileForward()
}
// SeekLT implements keyspan.FragmentIterator.
func (l *LevelIter) SeekLT(key []byte) *Span {
l.dir = -1
l.straddle = Span{}
l.straddleDir = 0
l.err = nil // clear cached iteration error
f := l.findFileLT(key)
if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(f.LargestRangeKey.UserKey, key) < 0 {
// Peek at the next file.
nextFile := l.files.Next()
l.files.Prev()
if nextFile != nil {
// We could unconditionally return an empty span between f.LargestRangeKey
// and the seek key, however if this span is to the right of all range keys
// on this level, it could lead to inconsistent behaviour in relative
// positioning operations. Consider this example, with a b-c range key:
//
// SeekLT(d) -> c-d:{}
// Prev() -> b-c{(#5,RANGEKEYSET,@4,foo)}
// Next() -> nil
//
// Iterators higher up in the iterator stack rely on this sort of relative
// positioning consistency.
//
// TODO(bilal): Investigate ways to be able to return straddle spans in
// cases similar to the above, while still retaining correctness.
// Return a straddling key instead of loading the file.
l.iterFile = f
if err := l.Close(); err != nil {
return l.verify(nil)
}
l.straddleDir = -1
l.straddle = Span{
Start: f.LargestRangeKey.UserKey,
End: nextFile.SmallestRangeKey.UserKey,
Keys: nil,
}
return l.verify(&l.straddle)
}
}
if l.loadFile(f, -1) == noFileLoaded {
return l.verify(nil)
}
if span := l.iter.SeekLT(key); span != nil {
return l.verify(span)
}
return l.skipEmptyFileBackward()
}
// First implements keyspan.FragmentIterator.
func (l *LevelIter) First() *Span {
l.dir = +1
l.straddle = Span{}
l.straddleDir = 0
l.err = nil // clear cached iteration error
if l.loadFile(l.files.First(), +1) == noFileLoaded {
return l.verify(nil)
}
if span := l.iter.First(); span != nil {
return l.verify(span)
}
return l.skipEmptyFileForward()
}
// Last implements keyspan.FragmentIterator.
func (l *LevelIter) Last() *Span {
l.dir = -1
l.straddle = Span{}
l.straddleDir = 0
l.err = nil // clear cached iteration error
if l.loadFile(l.files.Last(), -1) == noFileLoaded {
return l.verify(nil)
}
if span := l.iter.Last(); span != nil {
return l.verify(span)
}
return l.skipEmptyFileBackward()
}
// Next implements keyspan.FragmentIterator.
func (l *LevelIter) Next() *Span {
if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir > 0) {
return l.verify(nil)
}
if l.iter == nil && l.iterFile == nil {
// l.dir <= 0
return l.First()
}
l.dir = +1
if l.iter != nil {
if span := l.iter.Next(); span != nil {
return l.verify(span)
}
}
return l.skipEmptyFileForward()
}
// Prev implements keyspan.FragmentIterator.
func (l *LevelIter) Prev() *Span {
if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir < 0) {
return l.verify(nil)
}
if l.iter == nil && l.iterFile == nil {
// l.dir >= 0
return l.Last()
}
l.dir = -1
if l.iter != nil {
if span := l.iter.Prev(); span != nil {
return l.verify(span)
}
}
return l.skipEmptyFileBackward()
}
func (l *LevelIter) skipEmptyFileForward() *Span {
if l.straddleDir == 0 && l.keyType == manifest.KeyTypeRange &&
l.iterFile != nil && l.iter != nil {
// We were at a file that had spans. Check if the next file that has
// spans is not directly adjacent to the current file i.e. there is a
// gap in the span keyspace between the two files. In that case, synthesize
// a "straddle span" in l.straddle and return that.
//
// Straddle spans are not created in rangedel mode.
if err := l.Close(); err != nil {
l.err = err
return l.verify(nil)
}
startKey := l.iterFile.LargestRangeKey.UserKey
// Resetting l.iterFile without loading the file into l.iter is okay and
// does not change the logic in loadFile() as long as l.iter is also nil;
// which it should be due to the Close() call above.
l.iterFile = l.files.Next()
if l.iterFile == nil {
return l.verify(nil)
}
endKey := l.iterFile.SmallestRangeKey.UserKey
if l.cmp(startKey, endKey) < 0 {
// There is a gap between the two files. Synthesize a straddling span
// to avoid unnecessarily loading the next file.
l.straddle = Span{
Start: startKey,
End: endKey,
}
l.straddleDir = +1
return l.verify(&l.straddle)
}
} else if l.straddleDir < 0 {
// We were at a straddle key, but are now changing directions. l.iterFile
// was already moved backward by skipEmptyFileBackward, so advance it
// forward.
l.iterFile = l.files.Next()
}
l.straddle = Span{}
l.straddleDir = 0
var span *Span
for span.Empty() {
fileToLoad := l.iterFile
if l.keyType == manifest.KeyTypePoint {
// We haven't iterated to the next file yet if we're in point key
// (rangedel) mode.
fileToLoad = l.files.Next()
}
if l.loadFile(fileToLoad, +1) == noFileLoaded {
return l.verify(nil)
}
span = l.iter.First()
// In rangedel mode, we can expect to get empty files that we'd need to
// skip over, but not in range key mode.
if l.keyType == manifest.KeyTypeRange {
break
}
}
return l.verify(span)
}
func (l *LevelIter) skipEmptyFileBackward() *Span {
// We were at a file that had spans. Check if the previous file that has
// spans is not directly adjacent to the current file i.e. there is a
// gap in the span keyspace between the two files. In that case, synthesize
// a "straddle span" in l.straddle and return that.
//
// Straddle spans are not created in rangedel mode.
if l.straddleDir == 0 && l.keyType == manifest.KeyTypeRange &&
l.iterFile != nil && l.iter != nil {
if err := l.Close(); err != nil {
l.err = err
return l.verify(nil)
}
endKey := l.iterFile.SmallestRangeKey.UserKey
// Resetting l.iterFile without loading the file into l.iter is okay and
// does not change the logic in loadFile() as long as l.iter is also nil;
// which it should be due to the Close() call above.
l.iterFile = l.files.Prev()
if l.iterFile == nil {
return l.verify(nil)
}
startKey := l.iterFile.LargestRangeKey.UserKey
if l.cmp(startKey, endKey) < 0 {
// There is a gap between the two files. Synthesize a straddling span
// to avoid unnecessarily loading the next file.
l.straddle = Span{
Start: startKey,
End: endKey,
}
l.straddleDir = -1
return l.verify(&l.straddle)
}
} else if l.straddleDir > 0 {
// We were at a straddle key, but are now changing directions. l.iterFile
// was already advanced forward by skipEmptyFileForward, so move it
// backward.
l.iterFile = l.files.Prev()
}
l.straddle = Span{}
l.straddleDir = 0
var span *Span
for span.Empty() {
fileToLoad := l.iterFile
if l.keyType == manifest.KeyTypePoint {
fileToLoad = l.files.Prev()
}
if l.loadFile(fileToLoad, -1) == noFileLoaded {
return l.verify(nil)
}
span = l.iter.Last()
// In rangedel mode, we can expect to get empty files that we'd need to
// skip over, but not in range key mode as the filter on the FileMetadata
// should guarantee we always get a non-empty file.
if l.keyType == manifest.KeyTypeRange {
break
}
}
return l.verify(span)
}
// verify is invoked whenever a span is returned from an iterator positioning
// method to a caller. During invariant builds, it asserts invariants to the
// caller.
func (l *LevelIter) verify(s *Span) *Span {
// NB: Do not add any logic outside the invariants.Enabled conditional to
// ensure that verify is always compiled away in production builds.
if invariants.Enabled {
if f := l.files.Current(); f != l.iterFile {
panic(fmt.Sprintf("LevelIter.files.Current (%s) and l.iterFile (%s) diverged",
f, l.iterFile))
}
}
return s
}
// Error implements keyspan.FragmentIterator.
func (l *LevelIter) Error() error {
if l.err != nil || l.iter == nil {
return l.err
}
return l.iter.Error()
}
// Close implements keyspan.FragmentIterator.
func (l *LevelIter) Close() error {
if l.iter != nil {
l.err = l.iter.Close()
l.iter = nil
}
return l.err
}
// String implements keyspan.FragmentIterator.
func (l *LevelIter) String() string {
if l.iterFile != nil {
return fmt.Sprintf("%s: fileNum=%s", l.level, l.iterFile.FileNum)
}
return fmt.Sprintf("%s: fileNum=<nil>", l.level)
}

1209
vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

48
vendor/github.com/cockroachdb/pebble/internal/keyspan/seek.go generated vendored Normal file
View File

@@ -0,0 +1,48 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// SeekLE seeks to the span that contains or is before the target key.
func SeekLE(cmp base.Compare, iter FragmentIterator, key []byte) *Span {
// NB: We use SeekLT in order to land on the proper span for a search
// key that resides in the middle of a span. Consider the scenario:
//
// a---e
// e---i
//
// The spans are indexed by their start keys `a` and `e`. If the
// search key is `c` we want to land on the span [a,e). If we were to
// use SeekGE then the search key `c` would land on the span [e,i) and
// we'd have to backtrack. The one complexity here is what happens for the
// search key `e`. In that case SeekLT will land us on the span [a,e)
// and we'll have to move forward.
iterSpan := iter.SeekLT(key)
if iterSpan == nil {
// Advance the iterator once to see if the next span has a start key
// equal to key.
iterSpan = iter.Next()
if iterSpan == nil || cmp(key, iterSpan.Start) < 0 {
// The iterator is exhausted or we've hit the next span.
return nil
}
} else {
// Invariant: key > iterSpan.Start
if cmp(key, iterSpan.End) >= 0 {
// The current span lies entirely before the search key. Check to see if
// the next span contains the search key. If it doesn't, we'll backup
// and return to our earlier candidate.
iterSpan = iter.Next()
if iterSpan == nil || cmp(key, iterSpan.Start) < 0 {
// The next span is past our search key or there is no next span. Go
// back.
iterSpan = iter.Prev()
}
}
}
return iterSpan
}

467
vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go generated vendored Normal file
View File

@@ -0,0 +1,467 @@
// Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan // import "github.com/cockroachdb/pebble/internal/keyspan"
import (
"bytes"
"fmt"
"sort"
"strconv"
"strings"
"unicode"
"github.com/cockroachdb/pebble/internal/base"
)
// Span represents a set of keys over a span of user key space. All of the keys
// within a Span are applied across the span's key span indicated by Start and
// End. Each internal key applied over the user key span appears as a separate
// Key, with its own kind and sequence number. Optionally, each Key may also
// have a Suffix and/or Value.
//
// Note that the start user key is inclusive and the end user key is exclusive.
//
// Currently the only supported key kinds are:
//
// RANGEDEL, RANGEKEYSET, RANGEKEYUNSET, RANGEKEYDEL.
type Span struct {
// Start and End encode the user key range of all the contained items, with
// an inclusive start key and exclusive end key. Both Start and End must be
// non-nil, or both nil if representing an invalid Span.
Start, End []byte
// Keys holds the set of keys applied over the [Start, End) user key range.
// Keys is sorted by (SeqNum, Kind) descending, unless otherwise specified
// by the context. If SeqNum and Kind are equal, the order of Keys is
// undefined. Keys may be empty, even if Start and End are non-nil.
//
// Keys are a decoded representation of the internal keys stored in batches
// or sstable blocks. A single internal key in a range key block may produce
// several decoded Keys.
Keys []Key
KeysOrder KeysOrder
}
// KeysOrder describes the ordering of Keys within a Span.
type KeysOrder int8
const (
// ByTrailerDesc indicates a Span's keys are sorted by Trailer descending.
// This is the default ordering, and the ordering used during physical
// storage.
ByTrailerDesc KeysOrder = iota
// BySuffixAsc indicates a Span's keys are sorted by Suffix ascending. This
// ordering is used during user iteration of range keys.
BySuffixAsc
)
// Key represents a single key applied over a span of user keys. A Key is
// contained by a Span which specifies the span of user keys over which the Key
// is applied.
type Key struct {
// Trailer contains the key kind and sequence number.
Trailer uint64
// Suffix holds an optional suffix associated with the key. This is only
// non-nil for RANGEKEYSET and RANGEKEYUNSET keys.
Suffix []byte
// Value holds a logical value associated with the Key. It is NOT the
// internal value stored in a range key or range deletion block. This is
// only non-nil for RANGEKEYSET keys.
Value []byte
}
// SeqNum returns the sequence number component of the key.
func (k Key) SeqNum() uint64 {
return k.Trailer >> 8
}
// VisibleAt returns true if the provided key is visible at the provided
// snapshot sequence number. It interprets batch sequence numbers as always
// visible, because non-visible batch span keys are filtered when they're
// fragmented.
func (k Key) VisibleAt(snapshot uint64) bool {
seq := k.SeqNum()
return seq < snapshot || seq&base.InternalKeySeqNumBatch != 0
}
// Kind returns the kind component of the key.
func (k Key) Kind() base.InternalKeyKind {
return base.InternalKeyKind(k.Trailer & 0xff)
}
// Equal returns true if this Key is equal to the given key. Two keys are said
// to be equal if the two Keys have equal trailers, suffix and value. Suffix
// comparison uses the provided base.Compare func. Value comparison is bytewise.
func (k Key) Equal(equal base.Equal, b Key) bool {
return k.Trailer == b.Trailer &&
equal(k.Suffix, b.Suffix) &&
bytes.Equal(k.Value, b.Value)
}
// Valid returns true if the span is defined.
func (s *Span) Valid() bool {
return s.Start != nil && s.End != nil
}
// Empty returns true if the span does not contain any keys. An empty span may
// still be Valid. A non-empty span must be Valid.
//
// An Empty span may be produced by Visible, or be produced by iterators in
// order to surface the gaps between keys.
func (s *Span) Empty() bool {
return s == nil || len(s.Keys) == 0
}
// SmallestKey returns the smallest internal key defined by the span's keys.
// It requires the Span's keys be in ByTrailerDesc order. It panics if the span
// contains no keys or its keys are sorted in a different order.
func (s *Span) SmallestKey() base.InternalKey {
if len(s.Keys) == 0 {
panic("pebble: Span contains no keys")
} else if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
// The first key has the highest (sequence number,kind) tuple.
return base.InternalKey{
UserKey: s.Start,
Trailer: s.Keys[0].Trailer,
}
}
// LargestKey returns the largest internal key defined by the span's keys. The
// returned key will always be a "sentinel key" at the end boundary. The
// "sentinel key" models the exclusive end boundary by returning an InternalKey
// with the maximal sequence number, ensuring all InternalKeys with the same
// user key sort after the sentinel key.
//
// It requires the Span's keys be in ByTrailerDesc order. It panics if the span
// contains no keys or its keys are sorted in a different order.
func (s *Span) LargestKey() base.InternalKey {
if len(s.Keys) == 0 {
panic("pebble: Span contains no keys")
} else if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
// The last key has the lowest (sequence number,kind) tuple.
kind := s.Keys[len(s.Keys)-1].Kind()
return base.MakeExclusiveSentinelKey(kind, s.End)
}
// SmallestSeqNum returns the smallest sequence number of a key contained within
// the span. It requires the Span's keys be in ByTrailerDesc order. It panics if
// the span contains no keys or its keys are sorted in a different order.
func (s *Span) SmallestSeqNum() uint64 {
if len(s.Keys) == 0 {
panic("pebble: Span contains no keys")
} else if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
return s.Keys[len(s.Keys)-1].SeqNum()
}
// LargestSeqNum returns the largest sequence number of a key contained within
// the span. It requires the Span's keys be in ByTrailerDesc order. It panics if
// the span contains no keys or its keys are sorted in a different order.
func (s *Span) LargestSeqNum() uint64 {
if len(s.Keys) == 0 {
panic("pebble: Span contains no keys")
} else if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
return s.Keys[0].SeqNum()
}
// TODO(jackson): Replace most of the calls to Visible with more targeted calls
// that avoid the need to construct a new Span.
// Visible returns a span with the subset of keys visible at the provided
// sequence number. It requires the Span's keys be in ByTrailerDesc order. It
// panics if the span's keys are sorted in a different order.
//
// Visible may incur an allocation, so callers should prefer targeted,
// non-allocating methods when possible.
func (s Span) Visible(snapshot uint64) Span {
if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
ret := Span{Start: s.Start, End: s.End}
if len(s.Keys) == 0 {
return ret
}
// Keys from indexed batches may force an allocation. The Keys slice is
// ordered by sequence number, so ordinarily we can return the trailing
// subslice containing keys with sequence numbers less than `seqNum`.
//
// However, batch keys are special. Only visible batch keys are included
// when an Iterator's batch spans are fragmented. They must always be
// visible.
//
// Batch keys can create a sandwich of visible batch keys at the beginning
// of the slice and visible committed keys at the end of the slice, forcing
// us to allocate a new slice and copy the contents.
//
// Care is taking to only incur an allocation only when batch keys and
// visible keys actually sandwich non-visible keys.
// lastBatchIdx and lastNonVisibleIdx are set to the last index of a batch
// key and a non-visible key respectively.
lastBatchIdx := -1
lastNonVisibleIdx := -1
for i := range s.Keys {
if seqNum := s.Keys[i].SeqNum(); seqNum&base.InternalKeySeqNumBatch != 0 {
// Batch key. Always visible.
lastBatchIdx = i
} else if seqNum >= snapshot {
// This key is not visible.
lastNonVisibleIdx = i
}
}
// In the following comments: b = batch, h = hidden, v = visible (committed).
switch {
case lastNonVisibleIdx == -1:
// All keys are visible.
//
// [b b b], [v v v] and [b b b v v v]
ret.Keys = s.Keys
case lastBatchIdx == -1:
// There are no batch keys, so we can return the continuous subslice
// starting after the last non-visible Key.
//
// h h h [v v v]
ret.Keys = s.Keys[lastNonVisibleIdx+1:]
case lastNonVisibleIdx == len(s.Keys)-1:
// While we have a batch key and non-visible keys, there are no
// committed visible keys. The 'sandwich' is missing the bottom layer,
// so we can return the continuous sublice at the beginning.
//
// [b b b] h h h
ret.Keys = s.Keys[0 : lastBatchIdx+1]
default:
// This is the problematic sandwich case. Allocate a new slice, copying
// the batch keys and the visible keys into it.
//
// [b b b] h h h [v v v]
ret.Keys = make([]Key, (lastBatchIdx+1)+(len(s.Keys)-lastNonVisibleIdx-1))
copy(ret.Keys, s.Keys[:lastBatchIdx+1])
copy(ret.Keys[lastBatchIdx+1:], s.Keys[lastNonVisibleIdx+1:])
}
return ret
}
// VisibleAt returns true if the span contains a key visible at the provided
// snapshot. Keys with sequence numbers with the batch bit set are treated as
// always visible.
//
// VisibleAt requires the Span's keys be in ByTrailerDesc order. It panics if
// the span's keys are sorted in a different order.
func (s *Span) VisibleAt(snapshot uint64) bool {
if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
if len(s.Keys) == 0 {
return false
} else if first := s.Keys[0].SeqNum(); first&base.InternalKeySeqNumBatch != 0 {
// Only visible batch keys are included when an Iterator's batch spans
// are fragmented. They must always be visible.
return true
} else {
// Otherwise we check the last key. Since keys are ordered decreasing in
// sequence number, the last key has the lowest sequence number of any
// of the span's keys. If any of the keys are visible, the last key must
// be visible. Or put differently: if the last key is not visible, then
// no key is visible.
return s.Keys[len(s.Keys)-1].SeqNum() < snapshot
}
}
// ShallowClone returns the span with a Keys slice owned by the span itself.
// None of the key byte slices are cloned (see Span.DeepClone).
func (s *Span) ShallowClone() Span {
c := Span{
Start: s.Start,
End: s.End,
Keys: make([]Key, len(s.Keys)),
KeysOrder: s.KeysOrder,
}
copy(c.Keys, s.Keys)
return c
}
// DeepClone clones the span, creating copies of all contained slices. DeepClone
// is intended for non-production code paths like tests, the level checker, etc
// because it is allocation heavy.
func (s *Span) DeepClone() Span {
c := Span{
Start: make([]byte, len(s.Start)),
End: make([]byte, len(s.End)),
Keys: make([]Key, len(s.Keys)),
KeysOrder: s.KeysOrder,
}
copy(c.Start, s.Start)
copy(c.End, s.End)
for i := range s.Keys {
c.Keys[i].Trailer = s.Keys[i].Trailer
if len(s.Keys[i].Suffix) > 0 {
c.Keys[i].Suffix = make([]byte, len(s.Keys[i].Suffix))
copy(c.Keys[i].Suffix, s.Keys[i].Suffix)
}
if len(s.Keys[i].Value) > 0 {
c.Keys[i].Value = make([]byte, len(s.Keys[i].Value))
copy(c.Keys[i].Value, s.Keys[i].Value)
}
}
return c
}
// Contains returns true if the specified key resides within the span's bounds.
func (s *Span) Contains(cmp base.Compare, key []byte) bool {
return cmp(s.Start, key) <= 0 && cmp(key, s.End) < 0
}
// Covers returns true if the span covers keys at seqNum.
//
// Covers requires the Span's keys be in ByTrailerDesc order. It panics if the
// span's keys are sorted in a different order.
func (s Span) Covers(seqNum uint64) bool {
if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
return !s.Empty() && s.Keys[0].SeqNum() > seqNum
}
// CoversAt returns true if the span contains a key that is visible at the
// provided snapshot sequence number, and that key's sequence number is higher
// than seqNum.
//
// Keys with sequence numbers with the batch bit set are treated as always
// visible.
//
// CoversAt requires the Span's keys be in ByTrailerDesc order. It panics if the
// span's keys are sorted in a different order.
func (s *Span) CoversAt(snapshot, seqNum uint64) bool {
if s.KeysOrder != ByTrailerDesc {
panic("pebble: span's keys unexpectedly not in trailer order")
}
// NB: A key is visible at `snapshot` if its sequence number is strictly
// less than `snapshot`. See base.Visible.
for i := range s.Keys {
if kseq := s.Keys[i].SeqNum(); kseq&base.InternalKeySeqNumBatch != 0 {
// Only visible batch keys are included when an Iterator's batch spans
// are fragmented. They must always be visible.
return kseq > seqNum
} else if kseq < snapshot {
return kseq > seqNum
}
}
return false
}
// String returns a string representation of the span.
func (s Span) String() string {
return fmt.Sprint(prettySpan{Span: s, formatKey: base.DefaultFormatter})
}
// Pretty returns a formatter for the span.
func (s Span) Pretty(f base.FormatKey) fmt.Formatter {
// TODO(jackson): Take a base.FormatValue to format Key.Value too.
return prettySpan{s, f}
}
type prettySpan struct {
Span
formatKey base.FormatKey
}
func (s prettySpan) Format(fs fmt.State, c rune) {
if !s.Valid() {
fmt.Fprintf(fs, "<invalid>")
return
}
fmt.Fprintf(fs, "%s-%s:{", s.formatKey(s.Start), s.formatKey(s.End))
for i, k := range s.Keys {
if i > 0 {
fmt.Fprint(fs, " ")
}
fmt.Fprintf(fs, "(#%d,%s", k.SeqNum(), k.Kind())
if len(k.Suffix) > 0 || len(k.Value) > 0 {
fmt.Fprintf(fs, ",%s", k.Suffix)
}
if len(k.Value) > 0 {
fmt.Fprintf(fs, ",%s", k.Value)
}
fmt.Fprint(fs, ")")
}
fmt.Fprintf(fs, "}")
}
// SortKeysByTrailer sorts a keys slice by trailer.
func SortKeysByTrailer(keys *[]Key) {
// NB: keys is a pointer to a slice instead of a slice to avoid `sorted`
// escaping to the heap.
sorted := (*keysBySeqNumKind)(keys)
sort.Sort(sorted)
}
// KeysBySuffix implements sort.Interface, sorting its member Keys slice to by
// Suffix in the order dictated by Cmp.
type KeysBySuffix struct {
Cmp base.Compare
Keys []Key
}
func (s *KeysBySuffix) Len() int { return len(s.Keys) }
func (s *KeysBySuffix) Less(i, j int) bool { return s.Cmp(s.Keys[i].Suffix, s.Keys[j].Suffix) < 0 }
func (s *KeysBySuffix) Swap(i, j int) { s.Keys[i], s.Keys[j] = s.Keys[j], s.Keys[i] }
// ParseSpan parses the string representation of a Span. It's intended for
// tests. ParseSpan panics if passed a malformed span representation.
func ParseSpan(input string) Span {
var s Span
parts := strings.FieldsFunc(input, func(r rune) bool {
switch r {
case '-', ':', '{', '}':
return true
default:
return unicode.IsSpace(r)
}
})
s.Start, s.End = []byte(parts[0]), []byte(parts[1])
// Each of the remaining parts represents a single Key.
s.Keys = make([]Key, 0, len(parts)-2)
for _, p := range parts[2:] {
keyFields := strings.FieldsFunc(p, func(r rune) bool {
switch r {
case '#', ',', '(', ')':
return true
default:
return unicode.IsSpace(r)
}
})
var k Key
// Parse the sequence number.
seqNum, err := strconv.ParseUint(keyFields[0], 10, 64)
if err != nil {
panic(fmt.Sprintf("invalid sequence number: %q: %s", keyFields[0], err))
}
// Parse the key kind.
kind := base.ParseKind(keyFields[1])
k.Trailer = base.MakeTrailer(seqNum, kind)
// Parse the optional suffix.
if len(keyFields) >= 3 {
k.Suffix = []byte(keyFields[2])
}
// Parse the optional value.
if len(keyFields) >= 4 {
k.Value = []byte(keyFields[3])
}
s.Keys = append(s.Keys, k)
}
return s
}

50
vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go generated vendored Normal file
View File

@@ -0,0 +1,50 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// Transformer defines a transformation to be applied to a Span.
type Transformer interface {
// Transform takes a Span as input and writes the transformed Span to the
// provided output *Span pointer. The output Span's Keys slice may be reused
// by Transform to reduce allocations.
Transform(cmp base.Compare, in Span, out *Span) error
}
// The TransformerFunc type is an adapter to allow the use of ordinary functions
// as Transformers. If f is a function with the appropriate signature,
// TransformerFunc(f) is a Transformer that calls f.
type TransformerFunc func(base.Compare, Span, *Span) error
// Transform calls f(cmp, in, out).
func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
return tf(cmp, in, out)
}
var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = append(dst.Keys[:0], s.Keys...)
return nil
})
// VisibleTransform filters keys that are invisible at the provided snapshot
// sequence number.
func VisibleTransform(snapshot uint64) Transformer {
return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
dst.Start, dst.End = s.Start, s.End
dst.Keys = dst.Keys[:0]
for _, k := range s.Keys {
// NB: The InternalKeySeqNumMax value is used for the batch snapshot
// because a batch's visible span keys are filtered when they're
// fragmented. There's no requirement to enforce visibility at
// iteration time.
if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
dst.Keys = append(dst.Keys, k)
}
}
return nil
})
}

73
vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go generated vendored Normal file
View File

@@ -0,0 +1,73 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package keyspan
import "github.com/cockroachdb/pebble/internal/base"
// Truncate creates a new iterator where every span in the supplied iterator is
// truncated to be contained within the range [lower, upper). If start and end
// are specified, filter out any spans that are completely outside those bounds.
func Truncate(
cmp base.Compare,
iter FragmentIterator,
lower, upper []byte,
start, end *base.InternalKey,
panicOnUpperTruncate bool,
) FragmentIterator {
return Filter(iter, func(in *Span, out *Span) (keep bool) {
out.Start, out.End = in.Start, in.End
out.Keys = append(out.Keys[:0], in.Keys...)
// Ignore this span if it lies completely outside start, end. Note that
// end endInclusive indicated whether end is inclusive.
//
// The comparison between s.End and start is by user key only, as
// the span is exclusive at s.End, so comparing by user keys
// is sufficient.
if start != nil && cmp(in.End, start.UserKey) <= 0 {
return false
}
if end != nil {
v := cmp(in.Start, end.UserKey)
switch {
case v > 0:
// Wholly outside the end bound. Skip it.
return false
case v == 0:
// This span begins at the same user key as `end`. Whether or
// not any of the keys contained within the span are relevant is
// dependent on Trailers. Any keys contained within the span
// with trailers larger than end cover the small sliver of
// keyspace between [k#inf, k#<end-seqnum>]. Since keys are
// sorted descending by Trailer within the span, we need to find
// the prefix of keys with larger trailers.
for i := range in.Keys {
if in.Keys[i].Trailer < end.Trailer {
out.Keys = out.Keys[:i]
break
}
}
default:
// Wholly within the end bound. Keep it.
}
}
var truncated bool
// Truncate the bounds to lower and upper.
if cmp(in.Start, lower) < 0 {
out.Start = lower
}
if cmp(in.End, upper) > 0 {
truncated = true
out.End = upper
}
if panicOnUpperTruncate && truncated {
panic("pebble: upper bound should not be truncated")
}
return !out.Empty() && cmp(out.Start, out.End) < 0
})
}

1312
vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

2040
vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

46
vendor/github.com/cockroachdb/pebble/internal/manifest/level.go generated vendored Normal file
View File

@@ -0,0 +1,46 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package manifest
import "fmt"
const (
// 3 bits are necessary to represent level values from 0-6.
levelBits = 3
levelMask = (1 << levelBits) - 1
// invalidSublevel denotes an invalid or non-applicable sublevel.
invalidSublevel = -1
)
// Level encodes a level and optional sublevel for use in log and error
// messages. The encoding has the property that Level(0) ==
// L0Sublevel(invalidSublevel).
type Level uint32
func makeLevel(level, sublevel int) Level {
return Level(((sublevel + 1) << levelBits) | level)
}
// LevelToInt returns the int representation of a Level
func LevelToInt(l Level) int {
return int(l) & levelMask
}
// L0Sublevel returns a Level representing the specified L0 sublevel.
func L0Sublevel(sublevel int) Level {
if sublevel < 0 {
panic(fmt.Sprintf("invalid L0 sublevel: %d", sublevel))
}
return makeLevel(0, sublevel)
}
func (l Level) String() string {
level := int(l) & levelMask
sublevel := (int(l) >> levelBits) - 1
if sublevel != invalidSublevel {
return fmt.Sprintf("L%d.%d", level, sublevel)
}
return fmt.Sprintf("L%d", level)
}

704
vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go generated vendored Normal file
View File

@@ -0,0 +1,704 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package manifest
import (
"bytes"
"fmt"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
)
// LevelMetadata contains metadata for all of the files within
// a level of the LSM.
type LevelMetadata struct {
level int
totalSize uint64
tree btree
}
// clone makes a copy of the level metadata, implicitly increasing the ref
// count of every file contained within lm.
func (lm *LevelMetadata) clone() LevelMetadata {
return LevelMetadata{
level: lm.level,
totalSize: lm.totalSize,
tree: lm.tree.Clone(),
}
}
func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
return lm.tree.Release()
}
func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetadata {
bcmp := btreeCmpSeqNum
if level > 0 {
bcmp = btreeCmpSmallestKey(cmp)
}
var lm LevelMetadata
lm.level = level
lm.tree, _ = makeBTree(bcmp, files)
for _, f := range files {
lm.totalSize += f.Size
}
return lm
}
func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) {
var t btree
t.cmp = cmp
for _, f := range files {
t.Insert(f)
}
return t, newLevelSlice(t.Iter())
}
func (lm *LevelMetadata) insert(f *FileMetadata) error {
if err := lm.tree.Insert(f); err != nil {
return err
}
lm.totalSize += f.Size
return nil
}
func (lm *LevelMetadata) remove(f *FileMetadata) bool {
lm.totalSize -= f.Size
return lm.tree.Delete(f)
}
// Empty indicates whether there are any files in the level.
func (lm *LevelMetadata) Empty() bool {
return lm.tree.Count() == 0
}
// Len returns the number of files within the level.
func (lm *LevelMetadata) Len() int {
return lm.tree.Count()
}
// Size returns the cumulative size of all the files within the level.
func (lm *LevelMetadata) Size() uint64 {
return lm.totalSize
}
// Iter constructs a LevelIterator over the entire level.
func (lm *LevelMetadata) Iter() LevelIterator {
return LevelIterator{iter: lm.tree.Iter()}
}
// Slice constructs a slice containing the entire level.
func (lm *LevelMetadata) Slice() LevelSlice {
return newLevelSlice(lm.tree.Iter())
}
// Find finds the provided file in the level if it exists.
func (lm *LevelMetadata) Find(cmp base.Compare, m *FileMetadata) *LevelFile {
iter := lm.Iter()
if lm.level != 0 {
// If lm holds files for levels >0, we can narrow our search by binary
// searching by bounds.
o := overlaps(iter, cmp, m.Smallest.UserKey,
m.Largest.UserKey, m.Largest.IsExclusiveSentinel())
iter = o.Iter()
}
for f := iter.First(); f != nil; f = iter.Next() {
if f == m {
lf := iter.Take()
return &lf
}
}
return nil
}
// Annotation lazily calculates and returns the annotation defined by
// Annotator. The Annotator is used as the key for pre-calculated
// values, so equal Annotators must be used to avoid duplicate computations
// and cached annotations. Annotation must not be called concurrently, and in
// practice this is achieved by requiring callers to hold DB.mu.
func (lm *LevelMetadata) Annotation(annotator Annotator) interface{} {
if lm.Empty() {
return annotator.Zero(nil)
}
v, _ := lm.tree.root.Annotation(annotator)
return v
}
// InvalidateAnnotation clears any cached annotations defined by Annotator. The
// Annotator is used as the key for pre-calculated values, so equal Annotators
// must be used to clear the appropriate cached annotation. InvalidateAnnotation
// must not be called concurrently, and in practice this is achieved by
// requiring callers to hold DB.mu.
func (lm *LevelMetadata) InvalidateAnnotation(annotator Annotator) {
if lm.Empty() {
return
}
lm.tree.root.InvalidateAnnotation(annotator)
}
// LevelFile holds a file's metadata along with its position
// within a level of the LSM.
type LevelFile struct {
*FileMetadata
slice LevelSlice
}
// Slice constructs a LevelSlice containing only this file.
func (lf LevelFile) Slice() LevelSlice {
return lf.slice
}
// NewLevelSliceSeqSorted constructs a LevelSlice over the provided files,
// sorted by the L0 sequence number sort order.
// TODO(jackson): Can we improve this interface or avoid needing to export
// a slice constructor like this?
func NewLevelSliceSeqSorted(files []*FileMetadata) LevelSlice {
tr, slice := makeBTree(btreeCmpSeqNum, files)
tr.Release()
slice.verifyInvariants()
return slice
}
// NewLevelSliceKeySorted constructs a LevelSlice over the provided files,
// sorted by the files smallest keys.
// TODO(jackson): Can we improve this interface or avoid needing to export
// a slice constructor like this?
func NewLevelSliceKeySorted(cmp base.Compare, files []*FileMetadata) LevelSlice {
tr, slice := makeBTree(btreeCmpSmallestKey(cmp), files)
tr.Release()
slice.verifyInvariants()
return slice
}
// NewLevelSliceSpecificOrder constructs a LevelSlice over the provided files,
// ordering the files by their order in the provided slice. It's used in
// tests.
// TODO(jackson): Update tests to avoid requiring this and remove it.
func NewLevelSliceSpecificOrder(files []*FileMetadata) LevelSlice {
tr, slice := makeBTree(btreeCmpSpecificOrder(files), files)
tr.Release()
slice.verifyInvariants()
return slice
}
// newLevelSlice constructs a new LevelSlice backed by iter.
func newLevelSlice(iter iterator) LevelSlice {
s := LevelSlice{iter: iter}
if iter.r != nil {
s.length = iter.r.subtreeCount
}
s.verifyInvariants()
return s
}
// newBoundedLevelSlice constructs a new LevelSlice backed by iter and bounded
// by the provided start and end bounds. The provided startBound and endBound
// iterators must be iterators over the same B-Tree. Both start and end bounds
// are inclusive.
func newBoundedLevelSlice(iter iterator, startBound, endBound *iterator) LevelSlice {
s := LevelSlice{
iter: iter,
start: startBound,
end: endBound,
}
if iter.valid() {
s.length = endBound.countLeft() - startBound.countLeft()
// NB: The +1 is a consequence of the end bound being inclusive.
if endBound.valid() {
s.length++
}
// NB: A slice that's empty due to its bounds may have an endBound
// positioned before the startBound due to the inclusive bounds.
// TODO(jackson): Consider refactoring the end boundary to be exclusive;
// it would simplify some areas (eg, here) and complicate others (eg,
// Reslice-ing to grow compactions).
if s.length < 0 {
s.length = 0
}
}
s.verifyInvariants()
return s
}
// LevelSlice contains a slice of the files within a level of the LSM.
// A LevelSlice is immutable once created, but may be used to construct a
// mutable LevelIterator over the slice's files.
//
// LevelSlices should be constructed through one of the existing constructors,
// not manually initialized.
type LevelSlice struct {
iter iterator
length int
// start and end form the inclusive bounds of a slice of files within a
// level of the LSM. They may be nil if the entire B-Tree backing iter is
// accessible.
start *iterator
end *iterator
}
func (ls LevelSlice) verifyInvariants() {
if invariants.Enabled {
i := ls.Iter()
var length int
for f := i.First(); f != nil; f = i.Next() {
length++
}
if ls.length != length {
panic(fmt.Sprintf("LevelSlice %s has length %d value; actual length is %d", ls, ls.length, length))
}
}
}
// Each invokes fn for each element in the slice.
func (ls LevelSlice) Each(fn func(*FileMetadata)) {
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
fn(f)
}
}
// String implements fmt.Stringer.
func (ls LevelSlice) String() string {
var buf bytes.Buffer
fmt.Fprintf(&buf, "%d files: ", ls.length)
ls.Each(func(f *FileMetadata) {
if buf.Len() > 0 {
fmt.Fprintf(&buf, " ")
}
fmt.Fprint(&buf, f)
})
return buf.String()
}
// Empty indicates whether the slice contains any files.
func (ls *LevelSlice) Empty() bool {
return emptyWithBounds(ls.iter, ls.start, ls.end)
}
// Iter constructs a LevelIterator that iterates over the slice.
func (ls *LevelSlice) Iter() LevelIterator {
return LevelIterator{
start: ls.start,
end: ls.end,
iter: ls.iter.clone(),
}
}
// Len returns the number of files in the slice. Its runtime is constant.
func (ls *LevelSlice) Len() int {
return ls.length
}
// SizeSum sums the size of all files in the slice. Its runtime is linear in
// the length of the slice.
func (ls *LevelSlice) SizeSum() uint64 {
var sum uint64
iter := ls.Iter()
for f := iter.First(); f != nil; f = iter.Next() {
sum += f.Size
}
return sum
}
// Reslice constructs a new slice backed by the same underlying level, with
// new start and end positions. Reslice invokes the provided function, passing
// two LevelIterators: one positioned to i's inclusive start and one
// positioned to i's inclusive end. The resliceFunc may move either iterator
// forward or backwards, including beyond the callee's original bounds to
// capture additional files from the underlying level. Reslice constructs and
// returns a new LevelSlice with the final bounds of the iterators after
// calling resliceFunc.
func (ls LevelSlice) Reslice(resliceFunc func(start, end *LevelIterator)) LevelSlice {
if ls.iter.r == nil {
return ls
}
var start, end LevelIterator
if ls.start == nil {
start.iter = ls.iter.clone()
start.iter.first()
} else {
start.iter = ls.start.clone()
}
if ls.end == nil {
end.iter = ls.iter.clone()
end.iter.last()
} else {
end.iter = ls.end.clone()
}
resliceFunc(&start, &end)
return newBoundedLevelSlice(start.iter.clone(), &start.iter, &end.iter)
}
// KeyType is used to specify the type of keys we're looking for in
// LevelIterator positioning operations. Files not containing any keys of the
// desired type are skipped.
type KeyType int8
const (
// KeyTypePointAndRange denotes a search among the entire keyspace, including
// both point keys and range keys. No sstables are skipped.
KeyTypePointAndRange KeyType = iota
// KeyTypePoint denotes a search among the point keyspace. SSTables with no
// point keys will be skipped. Note that the point keyspace includes rangedels.
KeyTypePoint
// KeyTypeRange denotes a search among the range keyspace. SSTables with no
// range keys will be skipped.
KeyTypeRange
)
type keyTypeAnnotator struct{}
var _ Annotator = keyTypeAnnotator{}
func (k keyTypeAnnotator) Zero(dst interface{}) interface{} {
var val *KeyType
if dst != nil {
val = dst.(*KeyType)
} else {
val = new(KeyType)
}
*val = KeyTypePoint
return val
}
func (k keyTypeAnnotator) Accumulate(m *FileMetadata, dst interface{}) (interface{}, bool) {
v := dst.(*KeyType)
switch *v {
case KeyTypePoint:
if m.HasRangeKeys {
*v = KeyTypePointAndRange
}
case KeyTypePointAndRange:
// Do nothing.
default:
panic("unexpected key type")
}
return v, true
}
func (k keyTypeAnnotator) Merge(src interface{}, dst interface{}) interface{} {
v := dst.(*KeyType)
srcVal := src.(*KeyType)
switch *v {
case KeyTypePoint:
if *srcVal == KeyTypePointAndRange {
*v = KeyTypePointAndRange
}
case KeyTypePointAndRange:
// Do nothing.
default:
panic("unexpected key type")
}
return v
}
// LevelIterator iterates over a set of files' metadata. Its zero value is an
// empty iterator.
type LevelIterator struct {
iter iterator
start *iterator
end *iterator
filter KeyType
}
func (i LevelIterator) String() string {
var buf bytes.Buffer
iter := i.iter.clone()
iter.first()
iter.prev()
if i.iter.pos == -1 {
fmt.Fprint(&buf, "(<start>)*")
}
iter.next()
for ; iter.valid(); iter.next() {
if buf.Len() > 0 {
fmt.Fprint(&buf, " ")
}
if i.start != nil && cmpIter(iter, *i.start) == 0 {
fmt.Fprintf(&buf, " [ ")
}
isCurrentPos := cmpIter(iter, i.iter) == 0
if isCurrentPos {
fmt.Fprint(&buf, " ( ")
}
fmt.Fprint(&buf, iter.cur().String())
if isCurrentPos {
fmt.Fprint(&buf, " )*")
}
if i.end != nil && cmpIter(iter, *i.end) == 0 {
fmt.Fprintf(&buf, " ]")
}
}
if i.iter.n != nil && i.iter.pos >= i.iter.n.count {
if buf.Len() > 0 {
fmt.Fprint(&buf, " ")
}
fmt.Fprint(&buf, "(<end>)*")
}
return buf.String()
}
// Clone copies the iterator, returning an independent iterator at the same
// position.
func (i *LevelIterator) Clone() LevelIterator {
if i.iter.r == nil {
return *i
}
// The start and end iterators are not cloned and are treated as
// immutable.
return LevelIterator{
iter: i.iter.clone(),
start: i.start,
end: i.end,
filter: i.filter,
}
}
// Current returns the item at the current iterator position.
//
// Current is deprecated. Callers should instead use the return value of a
// positioning operation.
func (i *LevelIterator) Current() *FileMetadata {
if !i.iter.valid() ||
(i.end != nil && cmpIter(i.iter, *i.end) > 0) ||
(i.start != nil && cmpIter(i.iter, *i.start) < 0) {
return nil
}
return i.iter.cur()
}
func (i *LevelIterator) empty() bool {
return emptyWithBounds(i.iter, i.start, i.end)
}
// Filter clones the iterator and sets the desired KeyType as the key to filter
// files on.
func (i *LevelIterator) Filter(keyType KeyType) LevelIterator {
l := i.Clone()
l.filter = keyType
return l
}
func emptyWithBounds(i iterator, start, end *iterator) bool {
// If i.r is nil, the iterator was constructed from an empty btree.
// If the end bound is before the start bound, the bounds represent an
// empty slice of the B-Tree.
return i.r == nil || (start != nil && end != nil && cmpIter(*end, *start) < 0)
}
// First seeks to the first file in the iterator and returns it.
func (i *LevelIterator) First() *FileMetadata {
if i.empty() {
return nil
}
if i.start != nil {
i.iter = i.start.clone()
} else {
i.iter.first()
}
if !i.iter.valid() {
return nil
}
return i.skipFilteredForward(i.iter.cur())
}
// Last seeks to the last file in the iterator and returns it.
func (i *LevelIterator) Last() *FileMetadata {
if i.empty() {
return nil
}
if i.end != nil {
i.iter = i.end.clone()
} else {
i.iter.last()
}
if !i.iter.valid() {
return nil
}
return i.skipFilteredBackward(i.iter.cur())
}
// Next advances the iterator to the next file and returns it.
func (i *LevelIterator) Next() *FileMetadata {
if i.iter.r == nil {
return nil
}
if invariants.Enabled && (i.iter.pos >= i.iter.n.count || (i.end != nil && cmpIter(i.iter, *i.end) > 0)) {
panic("pebble: cannot next forward-exhausted iterator")
}
i.iter.next()
if !i.iter.valid() {
return nil
}
return i.skipFilteredForward(i.iter.cur())
}
// Prev moves the iterator the previous file and returns it.
func (i *LevelIterator) Prev() *FileMetadata {
if i.iter.r == nil {
return nil
}
if invariants.Enabled && (i.iter.pos < 0 || (i.start != nil && cmpIter(i.iter, *i.start) < 0)) {
panic("pebble: cannot prev backward-exhausted iterator")
}
i.iter.prev()
if !i.iter.valid() {
return nil
}
return i.skipFilteredBackward(i.iter.cur())
}
// SeekGE seeks to the first file in the iterator's file set with a largest
// user key greater than or equal to the provided user key. The iterator must
// have been constructed from L1+, because it requires the underlying files to
// be sorted by user keys and non-overlapping.
func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata {
// TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey.
if i.iter.r == nil {
return nil
}
m := i.seek(func(m *FileMetadata) bool {
return cmp(m.Largest.UserKey, userKey) >= 0
})
if i.filter != KeyTypePointAndRange && m != nil {
b, ok := m.LargestBound(i.filter)
if !ok {
m = i.Next()
} else if c := cmp(b.UserKey, userKey); c < 0 || c == 0 && b.IsExclusiveSentinel() {
// This file does not contain any keys of the type ≥ lower. It
// should be filtered, even though it does contain point keys.
m = i.Next()
}
}
return i.skipFilteredForward(m)
}
// SeekLT seeks to the last file in the iterator's file set with a smallest
// user key less than the provided user key. The iterator must have been
// constructed from L1+, because it requires the underlying files to be sorted
// by user keys and non-overlapping.
func (i *LevelIterator) SeekLT(cmp Compare, userKey []byte) *FileMetadata {
// TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey.
if i.iter.r == nil {
return nil
}
i.seek(func(m *FileMetadata) bool {
return cmp(m.Smallest.UserKey, userKey) >= 0
})
m := i.Prev()
// Although i.Prev() guarantees that the current file contains keys of the
// relevant type, it doesn't guarantee that the keys of the relevant type
// are < userKey.
if i.filter != KeyTypePointAndRange && m != nil {
b, ok := m.SmallestBound(i.filter)
if !ok {
panic("unreachable")
}
if c := cmp(b.UserKey, userKey); c >= 0 {
// This file does not contain any keys of the type ≥ lower. It
// should be filtered, even though it does contain point keys.
m = i.Prev()
}
}
return i.skipFilteredBackward(m)
}
// skipFilteredForward takes the file metadata at the iterator's current
// position, and skips forward if the current key-type filter (i.filter)
// excludes the file. It skips until it finds an unfiltered file or exhausts the
// level. If lower is != nil, skipFilteredForward skips any files that do not
// contain keys with the provided key-type ≥ lower.
//
// skipFilteredForward also enforces the upper bound, returning nil if at any
// point the upper bound is exceeded.
func (i *LevelIterator) skipFilteredForward(meta *FileMetadata) *FileMetadata {
for meta != nil && !meta.ContainsKeyType(i.filter) {
i.iter.next()
if !i.iter.valid() {
meta = nil
} else {
meta = i.iter.cur()
}
}
if meta != nil && i.end != nil && cmpIter(i.iter, *i.end) > 0 {
// Exceeded upper bound.
meta = nil
}
return meta
}
// skipFilteredBackward takes the file metadata at the iterator's current
// position, and skips backward if the current key-type filter (i.filter)
// excludes the file. It skips until it finds an unfiltered file or exhausts the
// level. If upper is != nil, skipFilteredBackward skips any files that do not
// contain keys with the provided key-type < upper.
//
// skipFilteredBackward also enforces the lower bound, returning nil if at any
// point the lower bound is exceeded.
func (i *LevelIterator) skipFilteredBackward(meta *FileMetadata) *FileMetadata {
for meta != nil && !meta.ContainsKeyType(i.filter) {
i.iter.prev()
if !i.iter.valid() {
meta = nil
} else {
meta = i.iter.cur()
}
}
if meta != nil && i.start != nil && cmpIter(i.iter, *i.start) < 0 {
// Exceeded lower bound.
meta = nil
}
return meta
}
func (i *LevelIterator) seek(fn func(*FileMetadata) bool) *FileMetadata {
i.iter.seek(fn)
// i.iter.seek seeked in the unbounded underlying B-Tree. If the iterator
// has start or end bounds, we may have exceeded them. Reset to the bounds
// if necessary.
//
// NB: The LevelIterator and LevelSlice semantics require that a bounded
// LevelIterator/LevelSlice containing files x0, x1, ..., xn behave
// identically to an unbounded LevelIterator/LevelSlice of a B-Tree
// containing x0, x1, ..., xn. In other words, any files outside the
// LevelIterator's bounds should not influence the iterator's behavior.
// When seeking, this means a SeekGE that seeks beyond the end bound,
// followed by a Prev should return the last element within bounds.
if i.end != nil && cmpIter(i.iter, *i.end) > 0 {
i.iter = i.end.clone()
// Since seek(fn) positioned beyond i.end, we know there is nothing to
// return within bounds.
i.iter.next()
return nil
} else if i.start != nil && cmpIter(i.iter, *i.start) < 0 {
i.iter = i.start.clone()
}
if !i.iter.valid() {
return nil
}
return i.iter.cur()
}
// Take constructs a LevelFile containing the file at the iterator's current
// position. Take panics if the iterator is not currently positioned over a
// file.
func (i *LevelIterator) Take() LevelFile {
m := i.Current()
if m == nil {
panic("Take called on invalid LevelIterator")
}
// LevelSlice's start and end fields are immutable and are positioned to
// the same position for a LevelFile because they're inclusive, so we can
// share one iterator stack between the two bounds.
boundsIter := i.iter.clone()
s := newBoundedLevelSlice(i.iter.clone(), &boundsIter, &boundsIter)
return LevelFile{
FileMetadata: m,
slice: s,
}
}

1540
vendor/github.com/cockroachdb/pebble/internal/manifest/version.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

1112
vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

60
vendor/github.com/cockroachdb/pebble/internal/manual/manual.go generated vendored Normal file
View File

@@ -0,0 +1,60 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package manual
// #include <stdlib.h>
import "C"
import "unsafe"
// The go:linkname directives provides backdoor access to private functions in
// the runtime. Below we're accessing the throw function.
//go:linkname throw runtime.throw
func throw(s string)
// TODO(peter): Rather than relying an C malloc/free, we could fork the Go
// runtime page allocator and allocate large chunks of memory using mmap or
// similar.
// New allocates a slice of size n. The returned slice is from manually managed
// memory and MUST be released by calling Free. Failure to do so will result in
// a memory leak.
func New(n int) []byte {
if n == 0 {
return make([]byte, 0)
}
// We need to be conscious of the Cgo pointer passing rules:
//
// https://golang.org/cmd/cgo/#hdr-Passing_pointers
//
// ...
// Note: the current implementation has a bug. While Go code is permitted
// to write nil or a C pointer (but not a Go pointer) to C memory, the
// current implementation may sometimes cause a runtime error if the
// contents of the C memory appear to be a Go pointer. Therefore, avoid
// passing uninitialized C memory to Go code if the Go code is going to
// store pointer values in it. Zero out the memory in C before passing it
// to Go.
ptr := C.calloc(C.size_t(n), 1)
if ptr == nil {
// NB: throw is like panic, except it guarantees the process will be
// terminated. The call below is exactly what the Go runtime invokes when
// it cannot allocate memory.
throw("out of memory")
}
// Interpret the C pointer as a pointer to a Go array, then slice.
return (*[MaxArrayLen]byte)(unsafe.Pointer(ptr))[:n:n]
}
// Free frees the specified slice.
func Free(b []byte) {
if cap(b) != 0 {
if len(b) == 0 {
b = b[:cap(b)]
}
ptr := unsafe.Pointer(&b[0])
C.free(ptr)
}
}

13
vendor/github.com/cockroachdb/pebble/internal/manual/manual_32bit.go generated vendored Normal file
View File

@@ -0,0 +1,13 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build 386 || amd64p32 || arm || armbe || mips || mipsle || mips64p32 || mips64p32le || ppc || sparc
// +build 386 amd64p32 arm armbe mips mipsle mips64p32 mips64p32le ppc sparc
package manual
const (
// MaxArrayLen is a safe maximum length for slices on this architecture.
MaxArrayLen = 1<<31 - 1
)

13
vendor/github.com/cockroachdb/pebble/internal/manual/manual_64bit.go generated vendored Normal file
View File

@@ -0,0 +1,13 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 || riscv64
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 riscv64
package manual
const (
// MaxArrayLen is a safe maximum length for slices on this architecture.
MaxArrayLen = 1<<50 - 1
)

20
vendor/github.com/cockroachdb/pebble/internal/manual/manual_nocgo.go generated vendored Normal file
View File

@@ -0,0 +1,20 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build !cgo
// +build !cgo
package manual
// Provides versions of New and Free when cgo is not available (e.g. cross
// compilation).
// New allocates a slice of size n.
func New(n int) []byte {
return make([]byte, n)
}
// Free frees the specified slice.
func Free(b []byte) {
}

19
vendor/github.com/cockroachdb/pebble/internal/private/batch.go generated vendored Normal file
View File

@@ -0,0 +1,19 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package private
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
)
// BatchSort is a hook for constructing iterators over the point and range
// mutations contained in a batch in sorted order. It is intended for testing
// use only.
var BatchSort func(interface{}) (
points base.InternalIterator,
rangeDels keyspan.FragmentIterator,
rangeKeys keyspan.FragmentIterator,
)

29
vendor/github.com/cockroachdb/pebble/internal/private/sstable.go generated vendored Normal file
View File

@@ -0,0 +1,29 @@
// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package private
import "github.com/cockroachdb/pebble/internal/base"
// SSTableCacheOpts is a hook for specifying cache options to
// sstable.NewReader.
var SSTableCacheOpts func(cacheID uint64, fileNum base.DiskFileNum) interface{}
// SSTableRawTombstonesOpt is a sstable.Reader option for disabling
// fragmentation of the range tombstones returned by
// sstable.Reader.NewRangeDelIter(). Used by debug tools to get a raw view of
// the tombstones contained in an sstable.
var SSTableRawTombstonesOpt interface{}
// SSTableWriterDisableKeyOrderChecks is a hook for disabling the key ordering
// invariant check performed by sstable.Writer. It is intended for internal use
// only in the construction of invalid sstables for testing. See
// tool/make_test_sstables.go.
var SSTableWriterDisableKeyOrderChecks func(interface{})
// SSTableInternalProperties is a func(*sstable.Writer) *sstable.Properties
// function that allows Pebble-internal code to mutate properties that external
// sstable writers are not permitted to edit. It's an untyped interface{} to
// avoid a cyclic dependency.
var SSTableInternalProperties interface{}

43
vendor/github.com/cockroachdb/pebble/internal/rangedel/rangedel.go generated vendored Normal file
View File

@@ -0,0 +1,43 @@
// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package rangedel
import (
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
)
// Encode takes a Span containing only range deletions. It invokes the provided
// closure with the encoded internal keys that represent the Span's state. The
// keys and values passed to emit are only valid until the closure returns. If
// emit returns an error, Encode stops and returns the error.
func Encode(s *keyspan.Span, emit func(k base.InternalKey, v []byte) error) error {
for _, k := range s.Keys {
if k.Kind() != base.InternalKeyKindRangeDelete {
return base.CorruptionErrorf("pebble: rangedel.Encode cannot encode %s key", k.Kind())
}
ik := base.InternalKey{
UserKey: s.Start,
Trailer: k.Trailer,
}
if err := emit(ik, s.End); err != nil {
return err
}
}
return nil
}
// Decode takes an internal key pair encoding a range deletion and returns a
// decoded keyspan containing the key. If keysDst is provided, the key will be
// appended to keysDst, avoiding an allocation.
func Decode(ik base.InternalKey, v []byte, keysDst []keyspan.Key) keyspan.Span {
return keyspan.Span{
Start: ik.UserKey,
End: v,
Keys: append(keysDst, keyspan.Key{
Trailer: ik.Trailer,
}),
}
}

369
vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go generated vendored Normal file
View File

@@ -0,0 +1,369 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
package rangekey
import (
"bytes"
"math"
"sort"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/pebble/internal/keyspan"
"github.com/cockroachdb/pebble/internal/manifest"
)
// UserIteratorConfig holds state for constructing the range key iterator stack
// for user iteration. The range key iterator must merge range key spans across
// the levels of the LSM. This merging is performed by a keyspan.MergingIter
// on-the-fly. The UserIteratorConfig implements keyspan.Transformer, evaluating
// range-key semantics and shadowing, so the spans returned by a MergingIter are
// fully resolved.
//
// The MergingIter is wrapped by a BoundedIter, which elides spans that are
// outside the iterator bounds (or the current prefix's bounds, during prefix
// iteration mode).
//
// To provide determinisim during iteration, the BoundedIter is wrapped by a
// DefragmentingIter that defragments abutting spans with identical
// user-observable state.
//
// At the top-level an InterleavingIter interleaves range keys with point keys
// and performs truncation to iterator bounds.
//
// Below is an abbreviated diagram illustrating the mechanics of a SeekGE.
//
// InterleavingIter.SeekGE
// │
// DefragmentingIter.SeekGE
// │
// BoundedIter.SeekGE
// │
// ╭────────────────┴───────────────╮
// │ ├── defragmentBwd*
// MergingIter.SeekGE │
// │ ╰── defragmentFwd
// ╰─╶╶ per level╶╶ ─╮
// │
// │
// ├── <?>.SeekLT
// │
// ╰── <?>.Next
type UserIteratorConfig struct {
snapshot uint64
comparer *base.Comparer
miter keyspan.MergingIter
biter keyspan.BoundedIter
diter keyspan.DefragmentingIter
liters [manifest.NumLevels]keyspan.LevelIter
litersUsed int
onlySets bool
bufs *Buffers
}
// Buffers holds various buffers used for range key iteration. They're exposed
// so that they may be pooled and reused between iterators.
type Buffers struct {
merging keyspan.MergingBuffers
defragmenting keyspan.DefragmentingBuffers
sortBuf keyspan.KeysBySuffix
}
// PrepareForReuse discards any excessively large buffers.
func (bufs *Buffers) PrepareForReuse() {
bufs.merging.PrepareForReuse()
bufs.defragmenting.PrepareForReuse()
}
// Init initializes the range key iterator stack for user iteration. The
// resulting fragment iterator applies range key semantics, defragments spans
// according to their user-observable state and, if onlySets = true, removes all
// Keys other than RangeKeySets describing the current state of range keys. The
// resulting spans contain Keys sorted by Suffix.
//
// The snapshot sequence number parameter determines which keys are visible. Any
// keys not visible at the provided snapshot are ignored.
func (ui *UserIteratorConfig) Init(
comparer *base.Comparer,
snapshot uint64,
lower, upper []byte,
hasPrefix *bool,
prefix *[]byte,
onlySets bool,
bufs *Buffers,
iters ...keyspan.FragmentIterator,
) keyspan.FragmentIterator {
ui.snapshot = snapshot
ui.comparer = comparer
ui.onlySets = onlySets
ui.miter.Init(comparer.Compare, ui, &bufs.merging, iters...)
ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix)
ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
ui.litersUsed = 0
ui.bufs = bufs
return &ui.diter
}
// AddLevel adds a new level to the bottom of the iterator stack. AddLevel
// must be called after Init and before any other method on the iterator.
func (ui *UserIteratorConfig) AddLevel(iter keyspan.FragmentIterator) {
ui.miter.AddLevel(iter)
}
// NewLevelIter returns a pointer to a newly allocated or reused
// keyspan.LevelIter. The caller is responsible for calling Init() on this
// instance.
func (ui *UserIteratorConfig) NewLevelIter() *keyspan.LevelIter {
if ui.litersUsed >= len(ui.liters) {
return &keyspan.LevelIter{}
}
ui.litersUsed++
return &ui.liters[ui.litersUsed-1]
}
// SetBounds propagates bounds to the iterator stack. The fragment iterator
// interface ordinarily doesn't enforce bounds, so this is exposed as an
// explicit method on the user iterator config.
func (ui *UserIteratorConfig) SetBounds(lower, upper []byte) {
ui.biter.SetBounds(lower, upper)
}
// Transform implements the keyspan.Transformer interface for use with a
// keyspan.MergingIter. It transforms spans by resolving range keys at the
// provided snapshot sequence number. Shadowing of keys is resolved (eg, removal
// of unset keys, removal of keys overwritten by a set at the same suffix, etc)
// and then non-RangeKeySet keys are removed. The resulting transformed spans
// only contain RangeKeySets describing the state visible at the provided
// sequence number, and hold their Keys sorted by Suffix.
func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error {
// Apply shadowing of keys.
dst.Start = s.Start
dst.End = s.End
ui.bufs.sortBuf = keyspan.KeysBySuffix{
Cmp: cmp,
Keys: ui.bufs.sortBuf.Keys[:0],
}
if err := coalesce(ui.comparer.Equal, &ui.bufs.sortBuf, ui.snapshot, s.Keys); err != nil {
return err
}
// During user iteration over range keys, unsets and deletes don't matter.
// Remove them if onlySets = true. This step helps logical defragmentation
// during iteration.
keys := ui.bufs.sortBuf.Keys
dst.Keys = dst.Keys[:0]
for i := range keys {
switch keys[i].Kind() {
case base.InternalKeyKindRangeKeySet:
if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
panic("pebble: keys unexpectedly not in ascending suffix order")
}
dst.Keys = append(dst.Keys, keys[i])
case base.InternalKeyKindRangeKeyUnset:
if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
panic("pebble: keys unexpectedly not in ascending suffix order")
}
if ui.onlySets {
// Skip.
continue
}
dst.Keys = append(dst.Keys, keys[i])
case base.InternalKeyKindRangeKeyDelete:
if ui.onlySets {
// Skip.
continue
}
dst.Keys = append(dst.Keys, keys[i])
default:
return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
}
}
// coalesce results in dst.Keys being sorted by Suffix.
dst.KeysOrder = keyspan.BySuffixAsc
return nil
}
// ShouldDefragment implements the DefragmentMethod interface and configures a
// DefragmentingIter to defragment spans of range keys if their user-visible
// state is identical. This defragmenting method assumes the provided spans have
// already been transformed through (UserIterationConfig).Transform, so all
// RangeKeySets are user-visible sets and are already in Suffix order. This
// defragmenter checks for equality between set suffixes and values (ignoring
// sequence numbers). It's intended for use during user iteration, when the
// wrapped keyspan iterator is merging spans across all levels of the LSM.
func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.Span) bool {
// This implementation must only be used on spans that have transformed by
// ui.Transform. The transform applies shadowing, removes all keys besides
// the resulting Sets and sorts the keys by suffix. Since shadowing has been
// applied, each Set must set a unique suffix. If the two spans are
// equivalent, they must have the same number of range key sets.
if len(a.Keys) != len(b.Keys) || len(a.Keys) == 0 {
return false
}
if a.KeysOrder != keyspan.BySuffixAsc || b.KeysOrder != keyspan.BySuffixAsc {
panic("pebble: range key span's keys unexpectedly not in ascending suffix order")
}
ret := true
for i := range a.Keys {
if invariants.Enabled {
if ui.onlySets && (a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet) {
panic("pebble: unexpected non-RangeKeySet during defragmentation")
}
if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 ||
ui.comparer.Compare(b.Keys[i].Suffix, b.Keys[i-1].Suffix) < 0) {
panic("pebble: range keys not ordered by suffix during defragmentation")
}
}
if !equal(a.Keys[i].Suffix, b.Keys[i].Suffix) {
ret = false
break
}
if !bytes.Equal(a.Keys[i].Value, b.Keys[i].Value) {
ret = false
break
}
}
return ret
}
// Coalesce imposes range key semantics and coalesces range keys with the same
// bounds. Coalesce drops any keys shadowed by more recent sets, unsets or
// deletes. Coalesce modifies the provided span's Keys slice, reslicing the
// slice to remove dropped keys.
//
// Coalescence has subtle behavior with respect to sequence numbers. Coalesce
// depends on a keyspan.Span's Keys being sorted in sequence number descending
// order. The first key has the largest sequence number. The returned coalesced
// span includes only the largest sequence number. All other sequence numbers
// are forgotten. When a compaction constructs output range keys from a
// coalesced span, it produces at most one RANGEKEYSET, one RANGEKEYUNSET and
// one RANGEKEYDEL. Each one of these keys adopt the largest sequence number.
//
// This has the potentially surprising effect of 'promoting' a key to a higher
// sequence number. This is okay, because:
// - There are no other overlapping keys within the coalesced span of
// sequence numbers (otherwise they would be in the compaction, due to
// the LSM invariant).
// - Range key sequence numbers are never compared to point key sequence
// numbers. Range keys and point keys have parallel existences.
// - Compactions only coalesce within snapshot stripes.
//
// Additionally, internal range keys at the same sequence number have subtle
// mechanics:
// - RANGEKEYSETs shadow RANGEKEYUNSETs of the same suffix.
// - RANGEKEYDELs only apply to keys at lower sequence numbers.
//
// This is required for ingestion. Ingested sstables are assigned a single
// sequence number for the file, at which all of the file's keys are visible.
// The RANGEKEYSET, RANGEKEYUNSET and RANGEKEYDEL key kinds are ordered such
// that among keys with equal sequence numbers (thus ordered by their kinds) the
// keys do not affect one another. Ingested sstables are expected to be
// consistent with respect to the set/unset suffixes: A given suffix should be
// set or unset but not both.
//
// The resulting dst Keys slice is sorted by Trailer.
func Coalesce(cmp base.Compare, eq base.Equal, keys []keyspan.Key, dst *[]keyspan.Key) error {
// TODO(jackson): Currently, Coalesce doesn't actually perform the sequence
// number promotion described in the comment above.
keysBySuffix := keyspan.KeysBySuffix{
Cmp: cmp,
Keys: (*dst)[:0],
}
if err := coalesce(eq, &keysBySuffix, math.MaxUint64, keys); err != nil {
return err
}
// Update the span with the (potentially reduced) keys slice. coalesce left
// the keys in *dst sorted by suffix. Re-sort them by trailer.
*dst = keysBySuffix.Keys
keyspan.SortKeysByTrailer(dst)
return nil
}
func coalesce(
equal base.Equal, keysBySuffix *keyspan.KeysBySuffix, snapshot uint64, keys []keyspan.Key,
) error {
// First, enforce visibility and RangeKeyDelete mechanics. We only need to
// consider the prefix of keys before and including the first
// RangeKeyDelete. We also must skip any keys that aren't visible at the
// provided snapshot sequence number.
//
// NB: Within a given sequence number, keys are ordered as:
// RangeKeySet > RangeKeyUnset > RangeKeyDelete
// This is significant, because this ensures that a Set or Unset sharing a
// sequence number with a Delete do not shadow each other.
deleteIdx := -1
for i := range keys {
if invariants.Enabled && i > 0 && keys[i].Trailer > keys[i-1].Trailer {
panic("pebble: invariant violation: span keys unordered")
}
if !keys[i].VisibleAt(snapshot) {
continue
}
// Once a RangeKeyDelete is observed, we know it shadows all subsequent
// keys and we can break early. We don't add the RangeKeyDelete key to
// keysBySuffix.keys yet, because we don't want a suffix-less key
// that appeared earlier in the slice to elide it. It'll be added back
// in at the end.
if keys[i].Kind() == base.InternalKeyKindRangeKeyDelete {
deleteIdx = i
break
}
keysBySuffix.Keys = append(keysBySuffix.Keys, keys[i])
}
// Sort the accumulated keys by suffix. There may be duplicates within a
// suffix, in which case the one with a larger trailer survives.
//
// We use a stable sort so that the first key with a given suffix is the one
// that with the highest Trailer (because the input `keys` was sorted by
// trailer descending).
sort.Stable(keysBySuffix)
// Grab a handle of the full sorted slice, before reslicing
// keysBySuffix.keys to accumulate the final coalesced keys.
sorted := keysBySuffix.Keys
keysBySuffix.Keys = keysBySuffix.Keys[:0]
var (
// prevSuffix is updated on each iteration of the below loop, and
// compared by the subsequent iteration to determine whether adjacent
// keys are defined at the same suffix.
prevSuffix []byte
// shadowing is set to true once any Key is shadowed by another key.
// When it's set to true—or after the loop if no keys are shadowed—the
// keysBySuffix.keys slice is resliced to contain the prefix of
// unshadowed keys. This avoids copying them incrementally in the common
// case of no shadowing.
shadowing bool
)
for i := range sorted {
if i > 0 && equal(prevSuffix, sorted[i].Suffix) {
// Skip; this key is shadowed by the predecessor that had a larger
// Trailer. If this is the first shadowed key, set shadowing=true
// and reslice keysBySuffix.keys to hold the entire unshadowed
// prefix.
if !shadowing {
keysBySuffix.Keys = keysBySuffix.Keys[:i]
shadowing = true
}
continue
}
prevSuffix = sorted[i].Suffix
if shadowing {
keysBySuffix.Keys = append(keysBySuffix.Keys, sorted[i])
}
}
// If there was no shadowing, keysBySuffix.keys is untouched. We can simply
// set it to the existing `sorted` slice (also backed by keysBySuffix.keys).
if !shadowing {
keysBySuffix.Keys = sorted
}
// If the original input `keys` slice contained a RangeKeyDelete, add it.
if deleteIdx >= 0 {
keysBySuffix.Keys = append(keysBySuffix.Keys, keys[deleteIdx])
}
return nil
}

411
vendor/github.com/cockroachdb/pebble/internal/rangekey/rangekey.go generated vendored Normal file
View File

@@ -0,0 +1,411 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
// Package rangekey provides facilities for encoding, decoding and merging range
// keys.
//
// Range keys map a span of keyspan `[start, end)`, at an optional suffix, to a
// value.
//
// # Encoding
//
// Unlike other Pebble keys, range keys encode several fields of information:
// start key, end key, suffix and value. Internally within Pebble and its
// sstables, all keys including range keys are represented as a key-value tuple.
// Range keys map to internal key-value tuples by mapping the start key to the
// key and encoding the remainder of the fields in the value.
//
// ## `RANGEKEYSET`
//
// A `RANGEKEYSET` represents one more range keys set over a single region of
// user key space. Each represented range key must have a unique suffix. A
// `RANGEKEYSET` encapsulates a start key, an end key and a set of SuffixValue
// pairs.
//
// A `RANGEKEYSET` key's user key holds the start key. Its value is a varstring
// end key, followed by a set of SuffixValue pairs. A `RANGEKEYSET` may have
// multiple SuffixValue pairs if the keyspan was set at multiple unique suffix
// values.
//
// ## `RANGEKEYUNSET`
//
// A `RANGEKEYUNSET` represents the removal of range keys at specific suffixes
// over a single region of user key space. A `RANGEKEYUNSET` encapsulates a
// start key, an end key and a set of suffixes.
//
// A `RANGEKEYUNSET` key's user key holds the start key. Its value is a
// varstring end key, followed by a set of suffixes. A `RANGEKEYUNSET` may have
// multiple suffixes if the keyspan was unset at multiple unique suffixes.
//
// ## `RANGEKEYDEL`
//
// A `RANGEKEYDEL` represents the removal of all range keys over a single region
// of user key space, regardless of suffix. A `RANGEKEYDEL` encapsulates a
// start key and an end key. The end key is stored in the value, without any
// varstring length prefixing.
package rangekey
// TODO(jackson): Document the encoding of RANGEKEYSET and RANGEKEYUNSET values
// once we're confident they're stable.
import (
"encoding/binary"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/keyspan"
)
// Encode takes a Span containing only range keys. It invokes the provided
// closure with the encoded internal keys that represent the Span's state. The
// keys and values passed to emit are only valid until the closure returns.
// If emit returns an error, Encode stops and returns the error.
func Encode(s *keyspan.Span, emit func(k base.InternalKey, v []byte) error) error {
enc := Encoder{Emit: emit}
return enc.Encode(s)
}
// An Encoder encodes range keys into their on-disk InternalKey format. An
// Encoder holds internal buffers, reused between Emit calls.
type Encoder struct {
Emit func(base.InternalKey, []byte) error
buf []byte
unsets [][]byte
sets []SuffixValue
}
// Encode takes a Span containing only range keys. It invokes the Encoder's Emit
// closure with the encoded internal keys that represent the Span's state. The
// keys and values passed to emit are only valid until the closure returns. If
// Emit returns an error, Encode stops and returns the error.
//
// The encoded key-value pair passed to Emit is only valid until the closure
// completes.
func (e *Encoder) Encode(s *keyspan.Span) error {
if s.Empty() {
return nil
}
// This for loop iterates through the span's keys, which are sorted by
// sequence number descending, grouping them into sequence numbers. All keys
// with identical sequence numbers are flushed together.
var del bool
var seqNum uint64
for i := range s.Keys {
if i == 0 || s.Keys[i].SeqNum() != seqNum {
if i > 0 {
// Flush all the existing internal keys that exist at seqNum.
if err := e.flush(s, seqNum, del); err != nil {
return err
}
}
// Reset sets, unsets, del.
seqNum = s.Keys[i].SeqNum()
del = false
e.sets = e.sets[:0]
e.unsets = e.unsets[:0]
}
switch s.Keys[i].Kind() {
case base.InternalKeyKindRangeKeySet:
e.sets = append(e.sets, SuffixValue{
Suffix: s.Keys[i].Suffix,
Value: s.Keys[i].Value,
})
case base.InternalKeyKindRangeKeyUnset:
e.unsets = append(e.unsets, s.Keys[i].Suffix)
case base.InternalKeyKindRangeKeyDelete:
del = true
default:
return base.CorruptionErrorf("pebble: %s key kind is not a range key", s.Keys[i].Kind())
}
}
return e.flush(s, seqNum, del)
}
// flush constructs internal keys for accumulated key state, and emits the
// internal keys.
func (e *Encoder) flush(s *keyspan.Span, seqNum uint64, del bool) error {
if len(e.sets) > 0 {
ik := base.MakeInternalKey(s.Start, seqNum, base.InternalKeyKindRangeKeySet)
l := EncodedSetValueLen(s.End, e.sets)
if l > cap(e.buf) {
e.buf = make([]byte, l)
}
EncodeSetValue(e.buf[:l], s.End, e.sets)
if err := e.Emit(ik, e.buf[:l]); err != nil {
return err
}
}
if len(e.unsets) > 0 {
ik := base.MakeInternalKey(s.Start, seqNum, base.InternalKeyKindRangeKeyUnset)
l := EncodedUnsetValueLen(s.End, e.unsets)
if l > cap(e.buf) {
e.buf = make([]byte, l)
}
EncodeUnsetValue(e.buf[:l], s.End, e.unsets)
if err := e.Emit(ik, e.buf[:l]); err != nil {
return err
}
}
if del {
ik := base.MakeInternalKey(s.Start, seqNum, base.InternalKeyKindRangeKeyDelete)
// s.End is stored directly in the value for RangeKeyDeletes.
if err := e.Emit(ik, s.End); err != nil {
return err
}
}
return nil
}
// Decode takes an internal key pair encoding range key(s) and returns a decoded
// keyspan containing the keys. If keysDst is provided, keys will be appended to
// keysDst.
func Decode(ik base.InternalKey, v []byte, keysDst []keyspan.Key) (keyspan.Span, error) {
var s keyspan.Span
// Hydrate the user key bounds.
s.Start = ik.UserKey
var ok bool
s.End, v, ok = DecodeEndKey(ik.Kind(), v)
if !ok {
return keyspan.Span{}, base.CorruptionErrorf("pebble: unable to decode range key end from %s", ik.Kind())
}
s.Keys = keysDst
// Hydrate the contents of the range key(s).
switch ik.Kind() {
case base.InternalKeyKindRangeKeySet:
for len(v) > 0 {
var sv SuffixValue
sv, v, ok = decodeSuffixValue(v)
if !ok {
return keyspan.Span{}, base.CorruptionErrorf("pebble: unable to decode range key suffix-value tuple")
}
s.Keys = append(s.Keys, keyspan.Key{
Trailer: ik.Trailer,
Suffix: sv.Suffix,
Value: sv.Value,
})
}
case base.InternalKeyKindRangeKeyUnset:
for len(v) > 0 {
var suffix []byte
suffix, v, ok = decodeSuffix(v)
if !ok {
return keyspan.Span{}, base.CorruptionErrorf("pebble: unable to decode range key unset suffix")
}
s.Keys = append(s.Keys, keyspan.Key{
Trailer: ik.Trailer,
Suffix: suffix,
})
}
case base.InternalKeyKindRangeKeyDelete:
if len(v) > 0 {
return keyspan.Span{}, base.CorruptionErrorf("pebble: RANGEKEYDELs must not contain additional data")
}
s.Keys = append(s.Keys, keyspan.Key{Trailer: ik.Trailer})
default:
return keyspan.Span{}, base.CorruptionErrorf("pebble: %s is not a range key", ik.Kind())
}
return s, nil
}
// SuffixValue represents a tuple of a suffix and a corresponding value. A
// physical RANGEKEYSET key may contain many logical RangeKeySets, each
// represented with a separate SuffixValue tuple.
type SuffixValue struct {
Suffix []byte
Value []byte
}
// encodedSetSuffixValuesLen precomputes the length of the given slice of
// SuffixValues, when encoded for a RangeKeySet. It may be used to construct a
// buffer of the appropriate size before encoding.
func encodedSetSuffixValuesLen(suffixValues []SuffixValue) int {
var n int
for i := 0; i < len(suffixValues); i++ {
n += lenVarint(len(suffixValues[i].Suffix))
n += len(suffixValues[i].Suffix)
n += lenVarint(len(suffixValues[i].Value))
n += len(suffixValues[i].Value)
}
return n
}
// encodeSetSuffixValues encodes a slice of SuffixValues for a RangeKeySet into
// dst. The length of dst must be greater than or equal to
// encodedSetSuffixValuesLen. encodeSetSuffixValues returns the number of bytes
// written, which should always equal the EncodedSetValueLen with the same
// arguments.
func encodeSetSuffixValues(dst []byte, suffixValues []SuffixValue) int {
// Encode the list of (suffix, value-len) tuples.
var n int
for i := 0; i < len(suffixValues); i++ {
// Encode the length of the suffix.
n += binary.PutUvarint(dst[n:], uint64(len(suffixValues[i].Suffix)))
// Encode the suffix itself.
n += copy(dst[n:], suffixValues[i].Suffix)
// Encode the value length.
n += binary.PutUvarint(dst[n:], uint64(len(suffixValues[i].Value)))
// Encode the value itself.
n += copy(dst[n:], suffixValues[i].Value)
}
return n
}
// EncodedSetValueLen precomputes the length of a RangeKeySet's value when
// encoded. It may be used to construct a buffer of the appropriate size before
// encoding.
func EncodedSetValueLen(endKey []byte, suffixValues []SuffixValue) int {
n := lenVarint(len(endKey))
n += len(endKey)
n += encodedSetSuffixValuesLen(suffixValues)
return n
}
// EncodeSetValue encodes a RangeKeySet's value into dst. The length of dst must
// be greater than or equal to EncodedSetValueLen. EncodeSetValue returns the
// number of bytes written, which should always equal the EncodedSetValueLen
// with the same arguments.
func EncodeSetValue(dst []byte, endKey []byte, suffixValues []SuffixValue) int {
// First encode the end key as a varstring.
n := binary.PutUvarint(dst, uint64(len(endKey)))
n += copy(dst[n:], endKey)
n += encodeSetSuffixValues(dst[n:], suffixValues)
return n
}
// DecodeEndKey reads the end key from the beginning of a range key (RANGEKEYSET,
// RANGEKEYUNSET or RANGEKEYDEL)'s physical encoded value. Both sets and unsets
// encode the range key, plus additional data in the value.
func DecodeEndKey(kind base.InternalKeyKind, data []byte) (endKey, value []byte, ok bool) {
switch kind {
case base.InternalKeyKindRangeKeyDelete:
// No splitting is necessary for range key deletes. The value is the end
// key, and there is no additional associated value.
return data, nil, true
case base.InternalKeyKindRangeKeySet, base.InternalKeyKindRangeKeyUnset:
v, n := binary.Uvarint(data)
if n <= 0 || uint64(n)+v >= uint64(len(data)) {
return nil, nil, false
}
endKey, value = data[n:n+int(v)], data[n+int(v):]
return endKey, value, true
default:
panic(errors.Newf("key kind %s is not a range key kind", kind))
}
}
// decodeSuffixValue decodes a single encoded SuffixValue from a RangeKeySet's
// split value. The end key must have already been stripped from the
// RangeKeySet's value (see DecodeEndKey).
func decodeSuffixValue(data []byte) (sv SuffixValue, rest []byte, ok bool) {
// Decode the suffix.
sv.Suffix, data, ok = decodeVarstring(data)
if !ok {
return SuffixValue{}, nil, false
}
// Decode the value.
sv.Value, data, ok = decodeVarstring(data)
if !ok {
return SuffixValue{}, nil, false
}
return sv, data, true
}
// encodedUnsetSuffixesLen precomputes the length of the given slice of
// suffixes, when encoded for a RangeKeyUnset. It may be used to construct a
// buffer of the appropriate size before encoding.
func encodedUnsetSuffixesLen(suffixes [][]byte) int {
var n int
for i := 0; i < len(suffixes); i++ {
n += lenVarint(len(suffixes[i]))
n += len(suffixes[i])
}
return n
}
// encodeUnsetSuffixes encodes a slice of suffixes for a RangeKeyUnset into dst.
// The length of dst must be greater than or equal to EncodedUnsetSuffixesLen.
// EncodeUnsetSuffixes returns the number of bytes written, which should always
// equal the EncodedUnsetSuffixesLen with the same arguments.
func encodeUnsetSuffixes(dst []byte, suffixes [][]byte) int {
// Encode the list of (suffix, value-len) tuples.
var n int
for i := 0; i < len(suffixes); i++ {
// Encode the length of the suffix.
n += binary.PutUvarint(dst[n:], uint64(len(suffixes[i])))
// Encode the suffix itself.
n += copy(dst[n:], suffixes[i])
}
return n
}
// EncodedUnsetValueLen precomputes the length of a RangeKeyUnset's value when
// encoded. It may be used to construct a buffer of the appropriate size before
// encoding.
func EncodedUnsetValueLen(endKey []byte, suffixes [][]byte) int {
n := lenVarint(len(endKey))
n += len(endKey)
n += encodedUnsetSuffixesLen(suffixes)
return n
}
// EncodeUnsetValue encodes a RangeKeyUnset's value into dst. The length of dst
// must be greater than or equal to EncodedUnsetValueLen. EncodeUnsetValue
// returns the number of bytes written, which should always equal the
// EncodedUnsetValueLen with the same arguments.
func EncodeUnsetValue(dst []byte, endKey []byte, suffixes [][]byte) int {
// First encode the end key as a varstring.
n := binary.PutUvarint(dst, uint64(len(endKey)))
n += copy(dst[n:], endKey)
n += encodeUnsetSuffixes(dst[n:], suffixes)
return n
}
// decodeSuffix decodes a single suffix from the beginning of data. If decoding
// suffixes from a RangeKeyUnset's value, the end key must have already been
// stripped from the RangeKeyUnset's value (see DecodeEndKey).
func decodeSuffix(data []byte) (suffix, rest []byte, ok bool) {
return decodeVarstring(data)
}
func decodeVarstring(data []byte) (v, rest []byte, ok bool) {
// Decode the length of the string.
l, n := binary.Uvarint(data)
if n <= 0 {
return nil, nil, ok
}
// Extract the string itself.
return data[n : n+int(l)], data[n+int(l):], true
}
// IsRangeKey returns true if the given key kind is one of the range key kinds.
func IsRangeKey(kind base.InternalKeyKind) bool {
switch kind {
case base.InternalKeyKindRangeKeyDelete,
base.InternalKeyKindRangeKeyUnset,
base.InternalKeyKindRangeKeySet:
return true
default:
return false
}
}
func lenVarint(v int) (n int) {
x := uint32(v)
n++
for x >= 0x80 {
x >>= 7
n++
}
return n
}

24
vendor/github.com/cockroachdb/pebble/internal/rawalloc/rawalloc.go generated vendored Normal file
View File

@@ -0,0 +1,24 @@
// Copyright 2018 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package rawalloc
// New returns a new byte slice of the specified length and capacity where the
// backing memory is uninitialized. This differs from make([]byte) which
// guarantees that the backing memory for the slice is initialized to zero. Use
// carefully.
func New(len, cap int) []byte {
ptr := mallocgc(uintptr(cap), nil, false)
return (*[maxArrayLen]byte)(ptr)[:len:cap]
}

22
vendor/github.com/cockroachdb/pebble/internal/rawalloc/rawalloc_32bit.go generated vendored Normal file
View File

@@ -0,0 +1,22 @@
// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//go:build 386 || amd64p32 || arm || armbe || mips || mipsle || mips64p32 || mips64p32le || ppc || sparc
// +build 386 amd64p32 arm armbe mips mipsle mips64p32 mips64p32le ppc sparc
package rawalloc
const (
maxArrayLen = 1<<31 - 1
)

22
vendor/github.com/cockroachdb/pebble/internal/rawalloc/rawalloc_64bit.go generated vendored Normal file
View File

@@ -0,0 +1,22 @@
// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 || riscv64
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 riscv64
package rawalloc
const (
maxArrayLen = 1<<50 - 1
)

23
vendor/github.com/cockroachdb/pebble/internal/rawalloc/rawalloc_gccgo.go generated vendored Normal file
View File

@@ -0,0 +1,23 @@
// Copyright 2018 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//go:build gccgo
// +build gccgo
package rawalloc
import "unsafe"
//extern runtime.mallocgc
func mallocgc(size uintptr, typ unsafe.Pointer, needzero bool) unsafe.Pointer

28
vendor/github.com/cockroachdb/pebble/internal/rawalloc/rawalloc_go1.9.go generated vendored Normal file
View File

@@ -0,0 +1,28 @@
// Copyright 2018 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//go:build gc && go1.9
// +build gc,go1.9
package rawalloc
import "unsafe"
// The go:linkname directives provides backdoor access to private functions in
// the runtime. Below we're accessing the mallocgc function. Note that this
// access is necessarily tied to a specific Go release which is why this file
// is protected by a build tag.
//go:linkname mallocgc runtime.mallocgc
func mallocgc(size uintptr, typ unsafe.Pointer, needzero bool) unsafe.Pointer

121
vendor/github.com/cockroachdb/pebble/internal/testkeys/strconv.go generated vendored Normal file
View File

@@ -0,0 +1,121 @@
/*
Copyright 2013 The Perkeep Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testkeys
import (
"strconv"
"github.com/cockroachdb/errors"
)
// parseUintBytes is like strconv.ParseUint, but using a []byte. Use of this
// function avoids an allocation when parsing an integer out of a []byte.
//
// This function is copied from go4.org/strconv.
func parseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) {
var cutoff, maxVal uint64
if bitSize == 0 {
bitSize = int(strconv.IntSize)
}
s0 := s
switch {
case len(s) < 1:
err = strconv.ErrSyntax
goto Error
case 2 <= base && base <= 36:
// valid base; nothing to do
case base == 0:
// Look for octal, hex prefix.
switch {
case s[0] == '0' && len(s) > 1 && (s[1] == 'x' || s[1] == 'X'):
base = 16
s = s[2:]
if len(s) < 1 {
err = strconv.ErrSyntax
goto Error
}
case s[0] == '0':
base = 8
default:
base = 10
}
default:
err = errors.New("invalid base " + strconv.Itoa(base))
goto Error
}
n = 0
cutoff = cutoff64(base)
maxVal = 1<<uint(bitSize) - 1
for i := 0; i < len(s); i++ {
var v byte
d := s[i]
switch {
case '0' <= d && d <= '9':
v = d - '0'
case 'a' <= d && d <= 'z':
v = d - 'a' + 10
case 'A' <= d && d <= 'Z':
v = d - 'A' + 10
default:
n = 0
err = strconv.ErrSyntax
goto Error
}
if int(v) >= base {
n = 0
err = strconv.ErrSyntax
goto Error
}
if n >= cutoff {
// n*base overflows
n = 1<<64 - 1
err = strconv.ErrRange
goto Error
}
n *= uint64(base)
n1 := n + uint64(v)
if n1 < n || n1 > maxVal {
// n+v overflows
n = 1<<64 - 1
err = strconv.ErrRange
goto Error
}
n = n1
}
return n, nil
Error:
return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
}
// Return the first number n such that n*base >= 1<<64.
func cutoff64(base int) uint64 {
if base < 2 {
return 0
}
return (1<<64-1)/uint64(base) + 1
}

480
vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go generated vendored Normal file
View File

@@ -0,0 +1,480 @@
// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
// Package testkeys provides facilities for generating and comparing
// human-readable test keys for use in tests and benchmarks. This package
// provides a single Comparer implementation that compares all keys generated
// by this package.
//
// Keys generated by this package may optionally have a 'suffix' encoding an
// MVCC timestamp. This suffix is of the form "@<integer>". Comparisons on the
// suffix are performed using integer value, not the byte representation.
package testkeys
import (
"bytes"
"fmt"
"math"
"strconv"
"strings"
"github.com/cockroachdb/pebble/internal/base"
"golang.org/x/exp/constraints"
"golang.org/x/exp/rand"
)
const alpha = "abcdefghijklmnopqrstuvwxyz"
const suffixDelim = '@'
var inverseAlphabet = make(map[byte]int64, len(alpha))
func init() {
for i := range alpha {
inverseAlphabet[alpha[i]] = int64(i)
}
}
// MaxSuffixLen is the maximum length of a suffix generated by this package.
var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", int64(math.MaxInt64)))
// Comparer is the comparer for test keys generated by this package.
var Comparer = &base.Comparer{
Compare: compare,
Equal: func(a, b []byte) bool { return compare(a, b) == 0 },
AbbreviatedKey: func(k []byte) uint64 {
return base.DefaultComparer.AbbreviatedKey(k[:split(k)])
},
FormatKey: base.DefaultFormatter,
Separator: func(dst, a, b []byte) []byte {
ai := split(a)
if ai == len(a) {
return append(dst, a...)
}
bi := split(b)
if bi == len(b) {
return append(dst, a...)
}
// If the keys are the same just return a.
if bytes.Equal(a[:ai], b[:bi]) {
return append(dst, a...)
}
n := len(dst)
dst = base.DefaultComparer.Separator(dst, a[:ai], b[:bi])
// Did it pick a separator different than a[:ai] -- if not we can't do better than a.
buf := dst[n:]
if bytes.Equal(a[:ai], buf) {
return append(dst[:n], a...)
}
// The separator is > a[:ai], so return it
return dst
},
Successor: func(dst, a []byte) []byte {
ai := split(a)
if ai == len(a) {
return append(dst, a...)
}
n := len(dst)
dst = base.DefaultComparer.Successor(dst, a[:ai])
// Did it pick a successor different than a[:ai] -- if not we can't do better than a.
buf := dst[n:]
if bytes.Equal(a[:ai], buf) {
return append(dst[:n], a...)
}
// The successor is > a[:ai], so return it.
return dst
},
ImmediateSuccessor: func(dst, a []byte) []byte {
// TODO(jackson): Consider changing this Comparer to only support
// representable prefix keys containing characters a-z.
ai := split(a)
if ai != len(a) {
panic("pebble: ImmediateSuccessor invoked with a non-prefix key")
}
return append(append(dst, a...), 0x00)
},
Split: split,
Name: "pebble.internal.testkeys",
}
func compare(a, b []byte) int {
ai, bi := split(a), split(b)
if v := bytes.Compare(a[:ai], b[:bi]); v != 0 {
return v
}
if len(a[ai:]) == 0 {
if len(b[bi:]) == 0 {
return 0
}
return -1
} else if len(b[bi:]) == 0 {
return +1
}
return compareTimestamps(a[ai:], b[bi:])
}
func split(a []byte) int {
i := bytes.LastIndexByte(a, suffixDelim)
if i >= 0 {
return i
}
return len(a)
}
func compareTimestamps(a, b []byte) int {
ai, err := parseUintBytes(bytes.TrimPrefix(a, []byte{suffixDelim}), 10, 64)
if err != nil {
panic(fmt.Sprintf("invalid test mvcc timestamp %q", a))
}
bi, err := parseUintBytes(bytes.TrimPrefix(b, []byte{suffixDelim}), 10, 64)
if err != nil {
panic(fmt.Sprintf("invalid test mvcc timestamp %q", b))
}
switch {
case ai < bi:
return +1
case ai > bi:
return -1
default:
return 0
}
}
// Keyspace describes a finite keyspace of unsuffixed test keys.
type Keyspace interface {
// Count returns the number of keys that exist within this keyspace.
Count() int64
// MaxLen returns the maximum length, in bytes, of a key within this
// keyspace. This is only guaranteed to return an upper bound.
MaxLen() int
// Slice returns the sub-keyspace from index i, inclusive, to index j,
// exclusive. The receiver is unmodified.
Slice(i, j int64) Keyspace
// EveryN returns a key space that includes 1 key for every N keys in the
// original keyspace. The receiver is unmodified.
EveryN(n int64) Keyspace
// key writes the i-th key to the buffer and returns the length.
key(buf []byte, i int64) int
}
// Divvy divides the provided keyspace into N equal portions, containing
// disjoint keys evenly distributed across the keyspace.
func Divvy(ks Keyspace, n int64) []Keyspace {
ret := make([]Keyspace, n)
for i := int64(0); i < n; i++ {
ret[i] = ks.Slice(i, ks.Count()).EveryN(n)
}
return ret
}
// Alpha constructs a keyspace consisting of all keys containing characters a-z,
// with at most `maxLength` characters.
func Alpha(maxLength int) Keyspace {
return alphabet{
alphabet: []byte(alpha),
maxLength: maxLength,
increment: 1,
}
}
// KeyAt returns the i-th key within the keyspace with a suffix encoding the
// timestamp t.
func KeyAt(k Keyspace, i int64, t int64) []byte {
b := make([]byte, k.MaxLen()+MaxSuffixLen)
return b[:WriteKeyAt(b, k, i, t)]
}
// WriteKeyAt writes the i-th key within the keyspace to the buffer dst, with a
// suffix encoding the timestamp t suffix. It returns the number of bytes
// written.
func WriteKeyAt(dst []byte, k Keyspace, i int64, t int64) int {
n := WriteKey(dst, k, i)
n += WriteSuffix(dst[n:], t)
return n
}
// Suffix returns the test keys suffix representation of timestamp t.
func Suffix(t int64) []byte {
b := make([]byte, MaxSuffixLen)
return b[:WriteSuffix(b, t)]
}
// SuffixLen returns the exact length of the given suffix when encoded.
func SuffixLen(t int64) int {
// Begin at 1 for the '@' delimiter, 1 for a single digit.
n := 2
t /= 10
for t > 0 {
t /= 10
n++
}
return n
}
// ParseSuffix returns the integer representation of the encoded suffix.
func ParseSuffix(s []byte) (int64, error) {
return strconv.ParseInt(strings.TrimPrefix(string(s), string(suffixDelim)), 10, 64)
}
// WriteSuffix writes the test keys suffix representation of timestamp t to dst,
// returning the number of bytes written.
func WriteSuffix(dst []byte, t int64) int {
dst[0] = suffixDelim
n := 1
n += len(strconv.AppendInt(dst[n:n], t, 10))
return n
}
// Key returns the i-th unsuffixed key within the keyspace.
func Key(k Keyspace, i int64) []byte {
b := make([]byte, k.MaxLen())
return b[:k.key(b, i)]
}
// WriteKey writes the i-th unsuffixed key within the keyspace to the buffer dst. It
// returns the number of bytes written.
func WriteKey(dst []byte, k Keyspace, i int64) int {
return k.key(dst, i)
}
type alphabet struct {
alphabet []byte
maxLength int
headSkip int64
tailSkip int64
increment int64
}
func (a alphabet) Count() int64 {
// Calculate the total number of keys, ignoring the increment.
total := keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip
// The increment dictates that we take every N keys, where N = a.increment.
// Consider a total containing the 5 keys:
// a b c d e
// ^ ^ ^
// If the increment is 2, this keyspace includes 'a', 'c' and 'e'. After
// dividing by the increment, there may be remainder. If there is, there's
// one additional key in the alphabet.
count := total / a.increment
if total%a.increment > 0 {
count++
}
return count
}
func (a alphabet) MaxLen() int {
return a.maxLength
}
func (a alphabet) Slice(i, j int64) Keyspace {
s := a
s.headSkip += i
s.tailSkip += a.Count() - j
return s
}
func (a alphabet) EveryN(n int64) Keyspace {
s := a
s.increment *= n
return s
}
func keyCount(n, l int) int64 {
if n == 0 {
return 0
} else if n == 1 {
return int64(l)
}
// The number of representable keys in the keyspace is a function of the
// length of the alphabet n and the max key length l. Consider how the
// number of representable keys grows as l increases:
//
// l = 1: n
// l = 2: n + n^2
// l = 3: n + n^2 + n^3
// ...
// Σ i=(1...l) n^i = n*(n^l - 1)/(n-1)
return (int64(n) * (int64(math.Pow(float64(n), float64(l))) - 1)) / int64(n-1)
}
func (a alphabet) key(buf []byte, idx int64) int {
// This function generates keys of length 1..maxKeyLength, pulling
// characters from the alphabet. The idx determines which key to generate,
// generating the i-th lexicographically next key.
//
// The index to use is advanced by `headSkip`, allowing a keyspace to encode
// a subregion of the keyspace.
//
// Eg, alphabet = `ab`, maxKeyLength = 3:
//
// aaa aab aba abb baa bab bba bbb
// aa ab ba bb
// a b
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13
//
return generateAlphabetKey(buf, a.alphabet, (idx*a.increment)+a.headSkip,
keyCount(len(a.alphabet), a.maxLength))
}
func generateAlphabetKey(buf, alphabet []byte, i, keyCount int64) int {
if keyCount == 0 || i > keyCount || i < 0 {
return 0
}
// Of the keyCount keys in the generative keyspace, how many are there
// starting with a particular character?
keysPerCharacter := keyCount / int64(len(alphabet))
// Find the character that the key at index i starts with and set it.
characterIdx := i / keysPerCharacter
buf[0] = alphabet[characterIdx]
// Consider characterIdx = 0, pointing to 'a'.
//
// aaa aab aba abb baa bab bba bbb
// aa ab ba bb
// a b
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13
// \_________________________/
// |keysPerCharacter| keys
//
// In our recursive call, we reduce the problem to:
//
// aaa aab aba abb
// aa ab
// 0 1 2 3 4 5
// \________________________/
// |keysPerCharacter-1| keys
//
// In the subproblem, there are keysPerCharacter-1 keys (eliminating the
// just 'a' key, plus any keys beginning with any other character).
//
// The index i is also offset, reduced by the count of keys beginning with
// characters earlier in the alphabet (keysPerCharacter*characterIdx) and
// the key consisting of just the 'a' (-1).
i = i - keysPerCharacter*characterIdx - 1
return 1 + generateAlphabetKey(buf[1:], alphabet, i, keysPerCharacter-1)
}
// computeAlphabetKeyIndex computes the inverse of generateAlphabetKey,
// returning the index of a particular key, given the provided alphabet and max
// length of a key.
//
// len(key) must be ≥ 1.
func computeAlphabetKeyIndex(key []byte, alphabet map[byte]int64, n int) int64 {
i, ok := alphabet[key[0]]
if !ok {
panic(fmt.Sprintf("unrecognized alphabet character %v", key[0]))
}
// How many keys exist that start with the preceding i characters? Each of
// the i characters themselves are a key, plus the count of all the keys
// with one less character for each.
ret := i + i*keyCount(len(alphabet), n-1)
if len(key) > 1 {
ret += 1 + computeAlphabetKeyIndex(key[1:], alphabet, n-1)
}
return ret
}
// RandomSeparator returns a random alphabetic key k such that a < k < b,
// pulling randomness from the provided random number generator. If dst is
// provided and the generated key fits within dst's capacity, the returned slice
// will use dst's memory.
//
// The keys a and b must be purely alphabetic, consisting only of characters
// a-z.
//
// If a prefix P exists such that Prefix(a) < P < Prefix(b), the generated key
// will consist of the prefix P appended with the provided suffix. A zero suffix
// generates an unsuffixed key. If no such prefix P exists, RandomSeparator will
// try to find a key k with either Prefix(a) or Prefix(b) such that a < k < b,
// but the generated key will not use the provided suffix. Note that it's
// possible that no separator key exists (eg, a='a@2', b='a@1'), in which case
// RandomSeparator returns nil.
//
// If RandomSeparator generates a new prefix, the generated prefix will have
// length at most MAX(maxLength, len(Prefix(a)), len(Prefix(b))).
//
// RandomSeparator panics if a or b fails to decode.
func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Rand) []byte {
// Determine both keys' logical prefixes and suffixes.
ai := Comparer.Split(a)
bi := Comparer.Split(b)
ap := a[:ai]
bp := b[:bi]
maxLength = max[int](maxLength, max[int](len(ap), len(bp)))
var as, bs int64
var err error
if ai != len(a) {
as, err = ParseSuffix(a[ai:])
if err != nil {
panic(fmt.Sprintf("failed to parse suffix of %q", a))
}
}
if bi != len(b) {
bs, err = ParseSuffix(b[bi:])
if err != nil {
panic(fmt.Sprintf("failed to parse suffix of %q", b))
}
}
apIdx := computeAlphabetKeyIndex(ap, inverseAlphabet, maxLength)
bpIdx := computeAlphabetKeyIndex(bp, inverseAlphabet, maxLength)
diff := bpIdx - apIdx
generatedIdx := bpIdx
if diff > 0 {
if as == 1 {
// There's no expressible key with suffix `a` greater than a@1, so
// exclude ap.
generatedIdx = rng.Int63n(diff) + apIdx + 1
} else {
generatedIdx = rng.Int63n(diff+1) + apIdx
}
}
switch {
case generatedIdx == apIdx && generatedIdx == bpIdx:
if bs-as <= 1 {
// There's no expressible suffix between the two, and there's no
// possible separator key.
return nil
}
// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
// any suffix we generate will be greater than it.
if (as != 0 && suffix >= as) || suffix <= bs {
suffix = rng.Int63n(bs-as-1) + as + 1
}
case generatedIdx == apIdx:
// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
// any suffix we generate will be greater than it.
if as != 0 && suffix >= as {
suffix = rng.Int63n(as)
}
case generatedIdx == bpIdx:
if suffix <= bs {
suffix = bs + rng.Int63n(10) + 1
}
}
if sz := maxLength + SuffixLen(suffix); cap(dst) < sz {
dst = make([]byte, sz)
} else {
dst = dst[:cap(dst)]
}
w := WriteKeyAt(dst, Alpha(maxLength), generatedIdx, suffix)
return dst[:w]
}
func max[I constraints.Ordered](a, b I) I {
if b > a {
return b
}
return a
}