aleesa-bot/aleesa-phrases-go
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Upgrade versions of vendored libs

Browse Source
This commit is contained in:
eleksir
2023-05-18 23:34:43 +03:00
parent 461ec78a7b
commit d90de40892
551 changed files with 36142 additions and 15606 deletions
Download Patch File Download Diff File Expand all files Collapse all files

209
vendor/github.com/cockroachdb/pebble/commit.go generated vendored
View File

@@ -8,6 +8,7 @@ import (
"runtime"
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/cockroachdb/pebble/record"
@@ -32,7 +33,7 @@ type commitQueue struct {
//
// The head index is stored in the most-significant bits so that we can
// atomically add to it and the overflow is harmless.
headTail uint64
headTail atomic.Uint64
// slots is a ring buffer of values stored in this queue. The size must be a
// power of 2. A slot is in use until *both* the tail index has moved beyond
@@ -57,10 +58,10 @@ func (q *commitQueue) pack(head, tail uint32) uint64 {
}
func (q *commitQueue) enqueue(b *Batch) {
ptrs := atomic.LoadUint64(&q.headTail)
ptrs := q.headTail.Load()
head, tail := q.unpack(ptrs)
if (tail+uint32(len(q.slots)))&(1<<dequeueBits-1) == head {
// Queue is full. This should never be reached because commitPipeline.sem
// Queue is full. This should never be reached because commitPipeline.commitQueueSem
// limits the number of concurrent operations.
panic("pebble: not reached")
}
@@ -79,12 +80,12 @@ func (q *commitQueue) enqueue(b *Batch) {
// Increment head. This passes ownership of slot to dequeue and acts as a
// store barrier for writing the slot.
atomic.AddUint64(&q.headTail, 1<<dequeueBits)
q.headTail.Add(1 << dequeueBits)
}
func (q *commitQueue) dequeue() *Batch {
for {
ptrs := atomic.LoadUint64(&q.headTail)
ptrs := q.headTail.Load()
head, tail := q.unpack(ptrs)
if tail == head {
// Queue is empty.
@@ -93,7 +94,7 @@ func (q *commitQueue) dequeue() *Batch {
slot := &q.slots[tail&uint32(len(q.slots)-1)]
b := (*Batch)(atomic.LoadPointer(slot))
if b == nil || atomic.LoadUint32(&b.applied) == 0 {
if b == nil || !b.applied.Load() {
// The batch is not ready to be dequeued, or another goroutine has
// already dequeued it.
return nil
@@ -102,7 +103,7 @@ func (q *commitQueue) dequeue() *Batch {
// Confirm head and tail (for our speculative check above) and increment
// tail. If this succeeds, then we own the slot at tail.
ptrs2 := q.pack(head, tail+1)
if atomic.CompareAndSwapUint64(&q.headTail, ptrs, ptrs2) {
if q.headTail.CompareAndSwap(ptrs, ptrs2) {
// We now own slot.
//
// Tell enqueue that we're done with this slot. Zeroing the slot is also
@@ -121,10 +122,10 @@ func (q *commitQueue) dequeue() *Batch {
type commitEnv struct {
// The next sequence number to give to a batch. Protected by
// commitPipeline.mu.
logSeqNum *uint64
logSeqNum *atomic.Uint64
// The visible sequence number at which reads should be performed. Ratcheted
// upwards atomically as batches are applied to the memtable.
visibleSeqNum *uint64
visibleSeqNum *atomic.Uint64
// Apply the batch to the specified memtable. Called concurrently.
apply func(b *Batch, mem *memTable) error
@@ -140,8 +141,8 @@ type commitEnv struct {
// (contained in a single Batch) atomically to the DB. The steps are
// conceptually:
//
// 1. Write the batch to the WAL and optionally sync the WAL
// 2. Apply the mutations in the batch to the memtable
// 1. Write the batch to the WAL and optionally sync the WAL
// 2. Apply the mutations in the batch to the memtable
//
// These two simple steps are made complicated by the desire for high
// performance. In the absence of concurrency, performance is limited by how
@@ -150,10 +151,10 @@ type commitEnv struct {
// pipeline. Performance under concurrency is the primary concern of the commit
// pipeline, though it also needs to maintain two invariants:
//
// 1. Batches need to be written to the WAL in sequence number order.
// 2. Batches need to be made visible for reads in sequence number order. This
// invariant arises from the use of a single sequence number which
// indicates which mutations are visible.
// 1. Batches need to be written to the WAL in sequence number order.
// 2. Batches need to be made visible for reads in sequence number order. This
// invariant arises from the use of a single sequence number which
// indicates which mutations are visible.
//
// Taking these invariants into account, let's revisit the work the commit
// pipeline needs to perform. Writing the batch to the WAL is necessarily
@@ -173,14 +174,14 @@ type commitEnv struct {
//
// The full outline of the commit pipeline operation is as follows:
//
// with commitPipeline mutex locked:
// assign batch sequence number
// write batch to WAL
// (optionally) add batch to WAL sync list
// apply batch to memtable (concurrently)
// wait for earlier batches to apply
// ratchet read sequence number
// (optionally) wait for the WAL to sync
// with commitPipeline mutex locked:
// assign batch sequence number
// write batch to WAL
// (optionally) add batch to WAL sync list
// apply batch to memtable (concurrently)
// wait for earlier batches to apply
// ratchet read sequence number
// (optionally) wait for the WAL to sync
//
// As soon as a batch has been written to the WAL, the commitPipeline mutex is
// released allowing another batch to write to the WAL. Each commit operation
@@ -217,7 +218,27 @@ type commitPipeline struct {
// Queue of pending batches to commit.
pending commitQueue
env commitEnv
sem chan struct{}
// The commit path has two queues:
// - commitPipeline.pending contains batches whose seqnums have not yet been
// published. It is a lock-free single producer multi consumer queue.
// - LogWriter.flusher.syncQ contains state for batches that have asked for
// a sync. It is a lock-free single producer single consumer queue.
// These lock-free queues have a fixed capacity. And since they are
// lock-free, we cannot do blocking waits when pushing onto these queues, in
// case they are full. Additionally, adding to these queues happens while
// holding commitPipeline.mu, and we don't want to block while holding that
// mutex since it is also needed by other code.
//
// Popping from these queues is independent and for a particular batch can
// occur in either order, though it is more common that popping from the
// commitPipeline.pending will happen first.
//
// Due to these constraints, we reserve a unit of space in each queue before
// acquiring commitPipeline.mu, which also ensures that the push operation
// is guaranteed to have space in the queue. The commitQueueSem and
// logSyncQSem are used for this reservation.
commitQueueSem chan struct{}
logSyncQSem chan struct{}
// The mutex to use for synchronizing access to logSeqNum and serializing
// calls to commitEnv.write().
mu sync.Mutex
@@ -226,23 +247,59 @@ type commitPipeline struct {
func newCommitPipeline(env commitEnv) *commitPipeline {
p := &commitPipeline{
env: env,
// The capacity of both commitQueue.slots and syncQueue.slots is set to
// record.SyncConcurrency, which also determines the value of these
// semaphores. We used to have a single semaphore, which required that the
// capacity of these queues be the same. Now that we have two semaphores,
// the capacity of these queues could be changed to be different. Say half
// of the batches asked to be synced, but syncing took 5x the latency of
// adding to the memtable and publishing. Then syncQueue.slots could be
// sized as 0.5*5 of the commitQueue.slots. We can explore this if we find
// that LogWriterMetrics.SyncQueueLen has high utilization under some
// workloads.
//
// NB: the commit concurrency is one less than SyncConcurrency because we
// have to allow one "slot" for a concurrent WAL rotation which will close
// and sync the WAL.
sem: make(chan struct{}, record.SyncConcurrency-1),
commitQueueSem: make(chan struct{}, record.SyncConcurrency-1),
logSyncQSem: make(chan struct{}, record.SyncConcurrency-1),
}
return p
}
// directWrite is used to directly write to the WAL. commitPipeline.mu must be
// held while this is called. DB.mu must not be held. directWrite will only
// return once the WAL sync is complete. Note that DirectWrite is a special case
// function which is currently only used when ingesting sstables as a flushable.
// Reason carefully about the correctness argument when calling this function
// from any context.
func (p *commitPipeline) directWrite(b *Batch) error {
var syncWG sync.WaitGroup
var syncErr error
syncWG.Add(1)
p.logSyncQSem <- struct{}{}
_, err := p.env.write(b, &syncWG, &syncErr)
syncWG.Wait()
err = firstError(err, syncErr)
return err
}
// Commit the specified batch, writing it to the WAL, optionally syncing the
// WAL, and applying the batch to the memtable. Upon successful return the
// batch's mutations will be visible for reading.
func (p *commitPipeline) Commit(b *Batch, syncWAL bool) error {
// REQUIRES: noSyncWait => syncWAL
func (p *commitPipeline) Commit(b *Batch, syncWAL bool, noSyncWait bool) error {
if b.Empty() {
return nil
}
p.sem <- struct{}{}
commitStartTime := time.Now()
// Acquire semaphores.
p.commitQueueSem <- struct{}{}
if syncWAL {
p.logSyncQSem <- struct{}{}
}
b.commitStats.SemaphoreWaitDuration = time.Since(commitStartTime)
// Prepare the batch for committing: enqueuing the batch in the pending
// queue, determining the batch sequence number and writing the data to the
@@ -250,27 +307,43 @@ func (p *commitPipeline) Commit(b *Batch, syncWAL bool) error {
//
// NB: We set Batch.commitErr on error so that the batch won't be a candidate
// for reuse. See Batch.release().
mem, err := p.prepare(b, syncWAL)
mem, err := p.prepare(b, syncWAL, noSyncWait)
if err != nil {
b.db = nil // prevent batch reuse on error
// NB: we are not doing <-p.commitQueueSem since the batch is still
// sitting in the pending queue. We should consider fixing this by also
// removing the batch from the pending queue.
return err
}
// Apply the batch to the memtable.
if err := p.env.apply(b, mem); err != nil {
b.db = nil // prevent batch reuse on error
// NB: we are not doing <-p.commitQueueSem since the batch is still
// sitting in the pending queue. We should consider fixing this by also
// removing the batch from the pending queue.
return err
}
// Publish the batch sequence number.
p.publish(b)
<-p.sem
<-p.commitQueueSem
if b.commitErr != nil {
b.db = nil // prevent batch reuse on error
if !noSyncWait {
// Already waited for commit, so look at the error.
if b.commitErr != nil {
b.db = nil // prevent batch reuse on error
err = b.commitErr
}
}
return b.commitErr
// Else noSyncWait. The LogWriter can be concurrently writing to
// b.commitErr. We will read b.commitErr in Batch.SyncWait after the
// LogWriter is done writing.
b.commitStats.TotalDuration = time.Since(commitStartTime)
return err
}
// AllocateSeqNum allocates count sequence numbers, invokes the prepare
@@ -280,7 +353,9 @@ func (p *commitPipeline) Commit(b *Batch, syncWAL bool) error {
// sstable ingestion within the commit pipeline. The prepare callback is
// invoked with commitPipeline.mu held, but note that DB.mu is not held and
// must be locked if necessary.
func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(seqNum uint64)) {
func (p *commitPipeline) AllocateSeqNum(
count int, prepare func(seqNum uint64), apply func(seqNum uint64),
) {
// This method is similar to Commit and prepare. Be careful about trying to
// share additional code with those methods because Commit and prepare are
// performance critical code paths.
@@ -294,7 +369,7 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se
b.setCount(uint32(count))
b.commit.Add(1)
p.sem <- struct{}{}
p.commitQueueSem <- struct{}{}
p.mu.Lock()
@@ -306,12 +381,12 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se
// Assign the batch a sequence number. Note that we use atomic operations
// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
// mutual exclusion for other goroutines writing to logSeqNum.
logSeqNum := atomic.AddUint64(p.env.logSeqNum, uint64(count)) - uint64(count)
logSeqNum := p.env.logSeqNum.Add(uint64(count)) - uint64(count)
seqNum := logSeqNum
if seqNum == 0 {
// We can't use the value 0 for the global seqnum during ingestion, because
// 0 indicates no global seqnum. So allocate one more seqnum.
atomic.AddUint64(p.env.logSeqNum, 1)
p.env.logSeqNum.Add(1)
seqNum++
}
b.setSeqNum(seqNum)
@@ -321,7 +396,7 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se
// writes that were sequenced before the ingestion. The spin loop is
// unfortunate, but obviates the need for additional synchronization.
for {
visibleSeqNum := atomic.LoadUint64(p.env.visibleSeqNum)
visibleSeqNum := p.env.visibleSeqNum.Load()
if visibleSeqNum == logSeqNum {
break
}
@@ -331,7 +406,7 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se
// Invoke the prepare callback. Note the lack of error reporting. Even if the
// callback internally fails, the sequence number needs to be published in
// order to allow the commit pipeline to proceed.
prepare()
prepare(b.SeqNum())
p.mu.Unlock()
@@ -341,26 +416,33 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se
// Publish the sequence number.
p.publish(b)
<-p.sem
<-p.commitQueueSem
}
func (p *commitPipeline) prepare(b *Batch, syncWAL bool) (*memTable, error) {
func (p *commitPipeline) prepare(b *Batch, syncWAL bool, noSyncWait bool) (*memTable, error) {
n := uint64(b.Count())
if n == invalidBatchCount {
return nil, ErrInvalidBatch
}
count := 1
if syncWAL {
count++
}
// count represents the waiting needed for publish, and optionally the
// waiting needed for the WAL sync.
b.commit.Add(count)
var syncWG *sync.WaitGroup
var syncErr *error
if syncWAL {
syncWG, syncErr = &b.commit, &b.commitErr
switch {
case !syncWAL:
// Only need to wait for the publish.
b.commit.Add(1)
// Remaining cases represent syncWAL=true.
case noSyncWait:
syncErr = &b.commitErr
syncWG = &b.fsyncWait
// Only need to wait synchronously for the publish. The user will
// (asynchronously) wait on the batch's fsyncWait.
b.commit.Add(1)
b.fsyncWait.Add(1)
case !noSyncWait:
syncErr = &b.commitErr
syncWG = &b.commit
// Must wait for both the publish and the WAL fsync.
b.commit.Add(2)
}
p.mu.Lock()
@@ -373,7 +455,7 @@ func (p *commitPipeline) prepare(b *Batch, syncWAL bool) (*memTable, error) {
// Assign the batch a sequence number. Note that we use atomic operations
// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
// mutual exclusion for other goroutines writing to logSeqNum.
b.setSeqNum(atomic.AddUint64(p.env.logSeqNum, n) - n)
b.setSeqNum(p.env.logSeqNum.Add(n) - n)
// Write the data to the WAL.
mem, err := p.env.write(b, syncWG, syncErr)
@@ -385,7 +467,7 @@ func (p *commitPipeline) prepare(b *Batch, syncWAL bool) (*memTable, error) {
func (p *commitPipeline) publish(b *Batch) {
// Mark the batch as applied.
atomic.StoreUint32(&b.applied, 1)
b.applied.Store(true)
// Loop dequeuing applied batches from the pending queue. If our batch was
// the head of the pending queue we are guaranteed that either we'll publish
@@ -399,10 +481,12 @@ func (p *commitPipeline) publish(b *Batch) {
if t == nil {
// Wait for another goroutine to publish us. We might also be waiting for
// the WAL sync to finish.
now := time.Now()
b.commit.Wait()
b.commitStats.CommitWaitDuration += time.Since(now)
break
}
if atomic.LoadUint32(&t.applied) != 1 {
if !t.applied.Load() {
panic("not reached")
}
@@ -411,13 +495,13 @@ func (p *commitPipeline) publish(b *Batch) {
// number for a subsequent batch. That's ok as all we're guaranteeing is
// that the sequence number ratchets up.
for {
curSeqNum := atomic.LoadUint64(p.env.visibleSeqNum)
curSeqNum := p.env.visibleSeqNum.Load()
newSeqNum := t.SeqNum() + uint64(t.Count())
if newSeqNum <= curSeqNum {
// t's sequence number has already been published.
break
}
if atomic.CompareAndSwapUint64(p.env.visibleSeqNum, curSeqNum, newSeqNum) {
if p.env.visibleSeqNum.CompareAndSwap(curSeqNum, newSeqNum) {
// We successfully published t's sequence number.
break
}
@@ -426,18 +510,3 @@ func (p *commitPipeline) publish(b *Batch) {
t.commit.Done()
}
}
// ratchetSeqNum allocates and marks visible all sequence numbers less than
// but excluding `nextSeqNum`.
func (p *commitPipeline) ratchetSeqNum(nextSeqNum uint64) {
p.mu.Lock()
defer p.mu.Unlock()
logSeqNum := atomic.LoadUint64(p.env.logSeqNum)
if logSeqNum >= nextSeqNum {
return
}
count := nextSeqNum - logSeqNum
_ = atomic.AddUint64(p.env.logSeqNum, uint64(count)) - uint64(count)
atomic.StoreUint64(p.env.visibleSeqNum, nextSeqNum)
}