aleesa-bot/aleesa-telegram-go
0
Fork 0
Code Activity

[WIP] lint a bit, update deps

Browse Source
This commit is contained in:
eleksir
2024-02-27 00:34:44 +03:00
parent 1892960896
commit 1f97424f31
201 changed files with 21474 additions and 9282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
vendor/github.com/cockroachdb/pebble/vfs/default_linux_noarm.go → vendor/github.com/cockroachdb/pebble/vfs/default_linux.go generated vendored
View File

@@ -2,8 +2,8 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build linux && !arm
// +build linux,!arm
//go:build linux
// +build linux
package vfs

73
vendor/github.com/cockroachdb/pebble/vfs/default_linux_arm.go generated vendored
View File

@@ -1,73 +0,0 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build linux && arm
// +build linux,arm
package vfs
import (
"os"
"syscall"
"github.com/cockroachdb/errors"
"golang.org/x/sys/unix"
)
func wrapOSFileImpl(f *os.File) File {
lf := &linuxOnArmFile{File: f, fd: f.Fd()}
if lf.fd != InvalidFd {
lf.useSyncRange = isSyncRangeSupported(lf.fd)
}
return lf
}
func (defaultFS) OpenDir(name string) (File, error) {
f, err := os.OpenFile(name, syscall.O_CLOEXEC, 0)
if err != nil {
return nil, errors.WithStack(err)
}
return &linuxOnArmDir{f}, nil
}
// Assert that linuxOnArmFile and linuxOnArmDir implement vfs.File.
var (
_ File = (*linuxOnArmDir)(nil)
_ File = (*linuxOnArmFile)(nil)
)
type linuxOnArmDir struct {
*os.File
}
func (d *linuxOnArmDir) Prefetch(offset int64, length int64) error { return nil }
func (d *linuxOnArmDir) Preallocate(offset, length int64) error { return nil }
func (d *linuxOnArmDir) SyncData() error { return d.Sync() }
func (d *linuxOnArmDir) SyncTo(offset int64) (fullSync bool, err error) { return false, nil }
type linuxOnArmFile struct {
*os.File
fd uintptr
useSyncRange bool
}
func (f *linuxOnArmFile) Prefetch(offset int64, length int64) error {
_, _, err := unix.Syscall(unix.SYS_READAHEAD, uintptr(f.fd), uintptr(offset), uintptr(length))
return err
}
func (f *linuxOnArmFile) Preallocate(offset, length int64) error { return nil }
func (f *linuxOnArmFile) SyncData() error {
// TODO(radu): does arm support unix.Fdatasync?
return f.Sync()
}
func (f *linuxOnArmFile) SyncTo(int64) (fullSync bool, err error) {
// syscall.SyncFileRange is not defined form arm (see https://github.com/cockroachdb/pebble/issues/171).
if err = f.Sync(); err != nil {
return false, err
}
return true, nil
}

161
vendor/github.com/cockroachdb/pebble/vfs/disk_full.go generated vendored
View File

@@ -39,68 +39,66 @@ func OnDiskFull(fs FS, fn func()) FS {
}
type enospcFS struct {
inner FS
atomic struct {
// generation is a monotonically increasing number that encodes the
// current state of ENOSPC error handling. Incoming writes are
// organized into generations to provide strong guarantees on when the
// disk full callback is invoked. The callback is invoked once per
// write generation.
//
// Special significance is given to the parity of this generation
// field to optimize incoming writes in the normal state, which only
// need to perform a single atomic load. If generation is odd, an
// ENOSPC error is being actively handled. The generations associated
// with writes are always even.
//
// The lifecycle of a write is:
//
// 1. Atomically load the current generation.
// a. If it's even, this is the write's generation number.
// b. If it's odd, an ENOSPC was recently encountered and the
// corresponding invocation of the disk full callback has not
// yet completed. The write must wait until the callback has
// completed and generation is updated to an even number, which
// becomes the write's generation number.
// 2. Perform the write. If it encounters no error or an error other
// than ENOSPC, the write returns and proceeds no further in this
// lifecycle.
// 3. Handle ENOSPC. If the write encounters ENOSPC, the callback must
// be invoked for the write's generation. The write's goroutine
// acquires the FS's mutex.
// a. If the FS's current generation is still equal to the write's
// generation, the write is the first write of its generation to
// encounter ENOSPC. It increments the FS's current generation
// to an odd number, signifying that an ENOSPC is being handled
// and invokes the callback.
// b. If the FS's current generation has changed, some other write
// from the same generation encountered an ENOSPC first. This
// write waits on the condition variable until the FS's current
// generation is updated indicating that the generation's
// callback invocation has completed.
// 3. Retry the write once. The callback for the write's generation
// has completed, either by this write's goroutine or another's.
// The write may proceed with the expectation that the callback
// remedied the full disk by freeing up disk space and an ENOSPC
// should not be encountered again for at least a few minutes. If
// we do encounter another ENOSPC on the retry, the callback was
// unable to remedy the full disk and another retry won't be
// useful. Any error, including ENOSPC, during the retry is
// returned without further handling. None of the retries invoke
// the callback.
//
// This scheme has a few nice properties:
// * Once the disk-full callback completes, it won't be invoked
// again unless a write that started strictly later encounters an
// ENOSPC. This is convenient if the callback strives to 'fix' the
// full disk, for example, by removing a ballast file. A new
// invocation of the callback guarantees a new problem.
// * Incoming writes block if there's an unhandled ENOSPC. Some
// writes, like WAL or MANIFEST fsyncs, are fatal if they encounter
// an ENOSPC.
generation uint32
}
mu struct {
inner FS
// generation is a monotonically increasing number that encodes the
// current state of ENOSPC error handling. Incoming writes are
// organized into generations to provide strong guarantees on when the
// disk full callback is invoked. The callback is invoked once per
// write generation.
//
// Special significance is given to the parity of this generation
// field to optimize incoming writes in the normal state, which only
// need to perform a single atomic load. If generation is odd, an
// ENOSPC error is being actively handled. The generations associated
// with writes are always even.
//
// The lifecycle of a write is:
//
// 1. Atomically load the current generation.
// a. If it's even, this is the write's generation number.
// b. If it's odd, an ENOSPC was recently encountered and the
// corresponding invocation of the disk full callback has not
// yet completed. The write must wait until the callback has
// completed and generation is updated to an even number, which
// becomes the write's generation number.
// 2. Perform the write. If it encounters no error or an error other
// than ENOSPC, the write returns and proceeds no further in this
// lifecycle.
// 3. Handle ENOSPC. If the write encounters ENOSPC, the callback must
// be invoked for the write's generation. The write's goroutine
// acquires the FS's mutex.
// a. If the FS's current generation is still equal to the write's
// generation, the write is the first write of its generation to
// encounter ENOSPC. It increments the FS's current generation
// to an odd number, signifying that an ENOSPC is being handled
// and invokes the callback.
// b. If the FS's current generation has changed, some other write
// from the same generation encountered an ENOSPC first. This
// write waits on the condition variable until the FS's current
// generation is updated indicating that the generation's
// callback invocation has completed.
// 3. Retry the write once. The callback for the write's generation
// has completed, either by this write's goroutine or another's.
// The write may proceed with the expectation that the callback
// remedied the full disk by freeing up disk space and an ENOSPC
// should not be encountered again for at least a few minutes. If
// we do encounter another ENOSPC on the retry, the callback was
// unable to remedy the full disk and another retry won't be
// useful. Any error, including ENOSPC, during the retry is
// returned without further handling. None of the retries invoke
// the callback.
//
// This scheme has a few nice properties:
// * Once the disk-full callback completes, it won't be invoked
// again unless a write that started strictly later encounters an
// ENOSPC. This is convenient if the callback strives to 'fix' the
// full disk, for example, by removing a ballast file. A new
// invocation of the callback guarantees a new problem.
// * Incoming writes block if there's an unhandled ENOSPC. Some
// writes, like WAL or MANIFEST fsyncs, are fatal if they encounter
// an ENOSPC.
generation atomic.Uint32
mu struct {
sync.Mutex
sync.Cond
onDiskFull func()
@@ -119,7 +117,7 @@ func (fs *enospcFS) Unwrap() FS {
// waitUntilReady blocks until the callback returns. The returned generation
// is always even.
func (fs *enospcFS) waitUntilReady() uint32 {
gen := atomic.LoadUint32(&fs.atomic.generation)
gen := fs.generation.Load()
if gen%2 == 0 {
// An even generation indicates that we're not currently handling an
// ENOSPC. Allow the write to proceed.
@@ -132,10 +130,10 @@ func (fs *enospcFS) waitUntilReady() uint32 {
defer fs.mu.Unlock()
// Load the generation again with fs.mu locked.
gen = atomic.LoadUint32(&fs.atomic.generation)
gen = fs.generation.Load()
for gen%2 == 1 {
fs.mu.Wait()
gen = atomic.LoadUint32(&fs.atomic.generation)
gen = fs.generation.Load()
}
return gen
}
@@ -144,7 +142,7 @@ func (fs *enospcFS) handleENOSPC(gen uint32) {
fs.mu.Lock()
defer fs.mu.Unlock()
currentGeneration := atomic.LoadUint32(&fs.atomic.generation)
currentGeneration := fs.generation.Load()
// If the current generation is still `gen`, this is the first goroutine
// to hit an ENOSPC within this write generation, so this goroutine is
@@ -153,7 +151,7 @@ func (fs *enospcFS) handleENOSPC(gen uint32) {
// Increment the generation to an odd number, indicating that the FS
// is out-of-disk space and incoming writes should pause and wait for
// the next generation before continuing.
atomic.StoreUint32(&fs.atomic.generation, gen+1)
fs.generation.Store(gen + 1)
func() {
// Drop the mutex while we invoke the callback, re-acquiring
@@ -165,7 +163,7 @@ func (fs *enospcFS) handleENOSPC(gen uint32) {
// Update the current generation again to an even number, indicating
// that the callback has completed for the write generation `gen`.
atomic.StoreUint32(&fs.atomic.generation, gen+2)
fs.generation.Store(gen + 2)
fs.mu.Broadcast()
return
}
@@ -185,7 +183,7 @@ func (fs *enospcFS) handleENOSPC(gen uint32) {
// subsequent generations' callbacks.
for currentGeneration == gen+1 {
fs.mu.Wait()
currentGeneration = atomic.LoadUint32(&fs.atomic.generation)
currentGeneration = fs.generation.Load()
}
}
@@ -230,6 +228,17 @@ func (fs *enospcFS) Open(name string, opts ...OpenOption) (File, error) {
return f, err
}
func (fs *enospcFS) OpenReadWrite(name string, opts ...OpenOption) (File, error) {
f, err := fs.inner.OpenReadWrite(name, opts...)
if f != nil {
f = &enospcFile{
fs: fs,
inner: f,
}
}
return f, err
}
func (fs *enospcFS) OpenDir(name string) (File, error) {
f, err := fs.inner.OpenDir(name)
if f != nil {
@@ -377,6 +386,20 @@ func (f *enospcFile) Write(p []byte) (n int, err error) {
return n, err
}
func (f *enospcFile) WriteAt(p []byte, ofs int64) (n int, err error) {
gen := f.fs.waitUntilReady()
n, err = f.inner.WriteAt(p, ofs)
if err != nil && isENOSPC(err) {
f.fs.handleENOSPC(gen)
var n2 int
n2, err = f.inner.WriteAt(p[n:], ofs+int64(n))
n += n2
}
return n, err
}
func (f *enospcFile) Prefetch(offset, length int64) error {
return f.inner.Prefetch(offset, length)
}

80
vendor/github.com/cockroachdb/pebble/vfs/disk_health.go generated vendored
View File

@@ -13,7 +13,6 @@ import (
"sync/atomic"
"time"
"github.com/cockroachdb/pebble/internal/invariants"
"github.com/cockroachdb/redact"
)
@@ -135,7 +134,7 @@ type diskHealthCheckingFile struct {
//
// NB: this packing scheme is not persisted, and is therefore safe to adjust
// across process boundaries.
lastWritePacked uint64
lastWritePacked atomic.Uint64
createTime time.Time
}
@@ -174,7 +173,7 @@ func (d *diskHealthCheckingFile) startTicker() {
return
case <-ticker.C:
packed := atomic.LoadUint64(&d.lastWritePacked)
packed := d.lastWritePacked.Load()
if packed == 0 {
continue
}
@@ -219,6 +218,14 @@ func (d *diskHealthCheckingFile) Write(p []byte) (n int, err error) {
return n, err
}
// Write implements the io.WriterAt interface.
func (d *diskHealthCheckingFile) WriteAt(p []byte, ofs int64) (n int, err error) {
d.timeDiskOp(OpTypeWrite, int64(len(p)), func() {
n, err = d.file.WriteAt(p, ofs)
})
return n, err
}
// Close implements the io.Closer interface.
func (d *diskHealthCheckingFile) Close() error {
d.stopTicker()
@@ -279,20 +286,12 @@ func (d *diskHealthCheckingFile) timeDiskOp(opType OpType, writeSizeInBytes int6
delta := time.Since(d.createTime)
packed := pack(delta, writeSizeInBytes, opType)
if invariants.Enabled {
if !atomic.CompareAndSwapUint64(&d.lastWritePacked, 0, packed) {
panic("concurrent write operations detected on file")
}
} else {
atomic.StoreUint64(&d.lastWritePacked, packed)
if d.lastWritePacked.Swap(packed) != 0 {
panic("concurrent write operations detected on file")
}
defer func() {
if invariants.Enabled {
if !atomic.CompareAndSwapUint64(&d.lastWritePacked, packed, 0) {
panic("concurrent write operations detected on file")
}
} else {
atomic.StoreUint64(&d.lastWritePacked, 0)
if d.lastWritePacked.Swap(0) != packed {
panic("concurrent write operations detected on file")
}
}()
op()
@@ -375,9 +374,17 @@ func (i DiskSlowInfo) String() string {
// SafeFormat implements redact.SafeFormatter.
func (i DiskSlowInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("disk slowness detected: %s on file %s (%d bytes) has been ongoing for %0.1fs",
redact.Safe(i.OpType.String()), redact.Safe(filepath.Base(i.Path)),
redact.Safe(i.WriteSize), redact.Safe(i.Duration.Seconds()))
switch i.OpType {
// Operations for which i.WriteSize is meaningful.
case OpTypeWrite, OpTypeSyncTo, OpTypePreallocate:
w.Printf("disk slowness detected: %s on file %s (%d bytes) has been ongoing for %0.1fs",
redact.Safe(i.OpType.String()), redact.Safe(filepath.Base(i.Path)),
redact.Safe(i.WriteSize), redact.Safe(i.Duration.Seconds()))
default:
w.Printf("disk slowness detected: %s on file %s has been ongoing for %0.1fs",
redact.Safe(i.OpType.String()), redact.Safe(filepath.Base(i.Path)),
redact.Safe(i.Duration.Seconds()))
}
}
// diskHealthCheckingFS adds disk-health checking facilities to a VFS.
@@ -443,7 +450,7 @@ type diskHealthCheckingFS struct {
type slot struct {
name string
opType OpType
startNanos int64
startNanos atomic.Int64
}
// diskHealthCheckingFS implements FS.
@@ -510,12 +517,12 @@ func (d *diskHealthCheckingFS) timeFilesystemOp(name string, opType OpType, op f
startNanos := time.Now().UnixNano()
for i := 0; i < len(d.mu.inflight); i++ {
if atomic.LoadInt64(&d.mu.inflight[i].startNanos) == 0 {
if d.mu.inflight[i].startNanos.Load() == 0 {
// This slot is not in use. Claim it.
s = d.mu.inflight[i]
s.name = name
s.opType = opType
atomic.StoreInt64(&s.startNanos, startNanos)
s.startNanos.Store(startNanos)
break
}
}
@@ -526,10 +533,10 @@ func (d *diskHealthCheckingFS) timeFilesystemOp(name string, opType OpType, op f
// incur an allocation.
if s == nil {
s = &slot{
name: name,
opType: opType,
startNanos: startNanos,
name: name,
opType: opType,
}
s.startNanos.Store(startNanos)
d.mu.inflight = append(d.mu.inflight, s)
}
}()
@@ -537,7 +544,7 @@ func (d *diskHealthCheckingFS) timeFilesystemOp(name string, opType OpType, op f
op()
// Signal completion by zeroing the start time.
atomic.StoreInt64(&s.startNanos, 0)
s.startNanos.Store(0)
}
// startTickerLocked starts a new goroutine with a ticker to monitor disk
@@ -548,7 +555,12 @@ func (d *diskHealthCheckingFS) startTickerLocked() {
go func() {
ticker := time.NewTicker(d.tickInterval)
defer ticker.Stop()
var exceededSlots []slot
type exceededSlot struct {
name string
opType OpType
startNanos int64
}
var exceededSlots []exceededSlot
for {
select {
@@ -559,14 +571,15 @@ func (d *diskHealthCheckingFS) startTickerLocked() {
d.mu.Lock()
now := time.Now()
for i := range d.mu.inflight {
nanos := atomic.LoadInt64(&d.mu.inflight[i].startNanos)
nanos := d.mu.inflight[i].startNanos.Load()
if nanos != 0 && time.Unix(0, nanos).Add(d.diskSlowThreshold).Before(now) {
// diskSlowThreshold was exceeded. Copy this inflightOp into
// exceededSlots and call d.onSlowDisk after dropping the mutex.
var inflightOp slot
inflightOp.name = d.mu.inflight[i].name
inflightOp.opType = d.mu.inflight[i].opType
inflightOp.startNanos = nanos
inflightOp := exceededSlot{
name: d.mu.inflight[i].name,
opType: d.mu.inflight[i].opType,
startNanos: nanos,
}
exceededSlots = append(exceededSlots, inflightOp)
}
}
@@ -680,6 +693,11 @@ func (d *diskHealthCheckingFS) Open(name string, opts ...OpenOption) (File, erro
return d.fs.Open(name, opts...)
}
// OpenReadWrite implements the FS interface.
func (d *diskHealthCheckingFS) OpenReadWrite(name string, opts ...OpenOption) (File, error) {
return d.fs.OpenReadWrite(name, opts...)
}
// OpenDir implements the FS interface.
func (d *diskHealthCheckingFS) OpenDir(name string) (File, error) {
f, err := d.fs.OpenDir(name)

26
vendor/github.com/cockroachdb/pebble/vfs/disk_usage_openbsd.go generated vendored Normal file
View File

@@ -0,0 +1,26 @@
// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build openbsd
// +build openbsd
package vfs
import "golang.org/x/sys/unix"
func (defaultFS) GetDiskUsage(path string) (DiskUsage, error) {
stat := unix.Statfs_t{}
if err := unix.Statfs(path, &stat); err != nil {
return DiskUsage{}, err
}
freeBytes := uint64(stat.F_bsize) * uint64(stat.F_bfree)
availBytes := uint64(stat.F_bsize) * uint64(stat.F_bavail)
totalBytes := uint64(stat.F_bsize) * uint64(stat.F_blocks)
return DiskUsage{
AvailBytes: availBytes,
TotalBytes: totalBytes,
UsedBytes: totalBytes - freeBytes,
}, nil
}

4
vendor/github.com/cockroachdb/pebble/vfs/disk_usage_unix.go generated vendored
View File

@@ -2,8 +2,8 @@
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//go:build darwin || openbsd || dragonfly || freebsd
// +build darwin openbsd dragonfly freebsd
//go:build darwin || dragonfly || freebsd
// +build darwin dragonfly freebsd
package vfs

33
vendor/github.com/cockroachdb/pebble/vfs/logging_fs.go generated vendored
View File

@@ -37,6 +37,24 @@ func (fs *loggingFS) Create(name string) (File, error) {
return newLoggingFile(f, name, fs.logFn), nil
}
func (fs *loggingFS) Open(name string, opts ...OpenOption) (File, error) {
fs.logFn("open: %s", name)
f, err := fs.FS.Open(name, opts...)
if err != nil {
return nil, err
}
return newLoggingFile(f, name, fs.logFn), nil
}
func (fs *loggingFS) OpenReadWrite(name string, opts ...OpenOption) (File, error) {
fs.logFn("open-read-write: %s", name)
f, err := fs.FS.OpenReadWrite(name, opts...)
if err != nil {
return nil, err
}
return newLoggingFile(f, name, fs.logFn), nil
}
func (fs *loggingFS) Link(oldname, newname string) error {
fs.logFn("link: %s -> %s", oldname, newname)
return fs.FS.Link(oldname, newname)
@@ -122,3 +140,18 @@ func (f *loggingFile) SyncTo(length int64) (fullSync bool, err error) {
f.logFn("sync-to(%d): %s", length, f.name)
return f.File.SyncTo(length)
}
func (f *loggingFile) ReadAt(p []byte, offset int64) (int, error) {
f.logFn("read-at(%d, %d): %s", offset, len(p), f.name)
return f.File.ReadAt(p, offset)
}
func (f *loggingFile) WriteAt(p []byte, offset int64) (int, error) {
f.logFn("write-at(%d, %d): %s", offset, len(p), f.name)
return f.File.WriteAt(p, offset)
}
func (f *loggingFile) Prefetch(offset int64, length int64) error {
f.logFn("prefetch(%d, %d): %s", offset, length, f.name)
return f.File.Prefetch(offset, length)
}

115
vendor/github.com/cockroachdb/pebble/vfs/mem_fs.go generated vendored
View File

@@ -14,6 +14,7 @@ import (
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/cockroachdb/errors"
@@ -63,7 +64,8 @@ func NewStrictMem() *MemFS {
// NewMemFile returns a memory-backed File implementation. The memory-backed
// file takes ownership of data.
func NewMemFile(data []byte) File {
n := &memNode{refs: 1}
n := &memNode{}
n.refs.Store(1)
n.mu.data = data
n.mu.modTime = time.Now()
return &memFile{
@@ -77,6 +79,11 @@ type MemFS struct {
mu sync.Mutex
root *memNode
// lockFiles holds a map of open file locks. Presence in this map indicates
// a file lock is currently held. Keys are strings holding the path of the
// locked file. The stored value is untyped and unused; only presence of
// the key within the map is significant.
lockedFiles sync.Map
strict bool
ignoreSyncs bool
// Windows has peculiar semantics with respect to hard links and deleting
@@ -208,6 +215,7 @@ func (y *MemFS) Create(fullname string) (File, error) {
ret = &memFile{
n: n,
fs: y,
read: true,
write: true,
}
}
@@ -216,7 +224,7 @@ func (y *MemFS) Create(fullname string) (File, error) {
if err != nil {
return nil, err
}
atomic.AddInt32(&ret.n.refs, 1)
ret.n.refs.Add(1)
return ret, nil
}
@@ -262,7 +270,7 @@ func (y *MemFS) Link(oldname, newname string) error {
})
}
func (y *MemFS) open(fullname string) (File, error) {
func (y *MemFS) open(fullname string, openForWrite bool) (File, error) {
var ret *memFile
err := y.walk(fullname, func(dir *memNode, frag string, final bool) error {
if final {
@@ -275,9 +283,10 @@ func (y *MemFS) open(fullname string) (File, error) {
}
if n := dir.children[frag]; n != nil {
ret = &memFile{
n: n,
fs: y,
read: true,
n: n,
fs: y,
read: true,
write: openForWrite,
}
}
}
@@ -293,18 +302,28 @@ func (y *MemFS) open(fullname string) (File, error) {
Err: oserror.ErrNotExist,
}
}
atomic.AddInt32(&ret.n.refs, 1)
ret.n.refs.Add(1)
return ret, nil
}
// Open implements FS.Open.
func (y *MemFS) Open(fullname string, opts ...OpenOption) (File, error) {
return y.open(fullname)
return y.open(fullname, false /* openForWrite */)
}
// OpenReadWrite implements FS.OpenReadWrite.
func (y *MemFS) OpenReadWrite(fullname string, opts ...OpenOption) (File, error) {
f, err := y.open(fullname, true /* openForWrite */)
pathErr, ok := err.(*os.PathError)
if ok && pathErr.Err == oserror.ErrNotExist {
return y.Create(fullname)
}
return f, err
}
// OpenDir implements FS.OpenDir.
func (y *MemFS) OpenDir(fullname string) (File, error) {
return y.open(fullname)
return y.open(fullname, false /* openForWrite */)
}
// Remove implements FS.Remove.
@@ -323,7 +342,7 @@ func (y *MemFS) Remove(fullname string) error {
// Windows semantics. This ensures that we don't regress in the
// ordering of operations and try to remove a file while it is
// still open.
if n := atomic.LoadInt32(&child.refs); n > 0 {
if n := child.refs.Load(); n > 0 {
return oserror.ErrInvalid
}
}
@@ -444,9 +463,31 @@ func (y *MemFS) MkdirAll(dirname string, perm os.FileMode) error {
// Lock implements FS.Lock.
func (y *MemFS) Lock(fullname string) (io.Closer, error) {
// FS.Lock excludes other processes, but other processes cannot see this
// process' memory. We translate Lock into Create so that have the normal
// detection of non-existent directory paths.
return y.Create(fullname)
// process' memory. However some uses (eg, Cockroach tests) may open and
// close the same MemFS-backed database multiple times. We want mutual
// exclusion in this case too. See cockroachdb/cockroach#110645.
_, loaded := y.lockedFiles.Swap(fullname, nil /* the value itself is insignificant */)
if loaded {
// This file lock has already been acquired. On unix, this results in
// either EACCES or EAGAIN so we mimic.
return nil, syscall.EAGAIN
}
// Otherwise, we successfully acquired the lock. Locks are visible in the
// parent directory listing, and they also must be created under an existent
// directory. Create the path so that we have the normal detection of
// non-existent directory paths, and make the lock visible when listing
// directory entries.
f, err := y.Create(fullname)
if err != nil {
// "Release" the lock since we failed.
y.lockedFiles.Delete(fullname)
return nil, err
}
return &memFileLock{
y: y,
f: f,
fullname: fullname,
}, nil
}
// List implements FS.List.
@@ -513,7 +554,7 @@ func (*MemFS) GetDiskUsage(string) (DiskUsage, error) {
type memNode struct {
name string
isDir bool
refs int32
refs atomic.Int32
// Mutable state.
// - For a file: data, syncedDate, modTime: A file is only being mutated by a single goroutine,
@@ -629,7 +670,7 @@ type memFile struct {
var _ File = (*memFile)(nil)
func (f *memFile) Close() error {
if n := atomic.AddInt32(&f.n.refs, -1); n < 0 {
if n := f.n.refs.Add(-1); n < 0 {
panic(fmt.Sprintf("pebble: close of unopened file: %d", n))
}
f.n = nil
@@ -665,7 +706,11 @@ func (f *memFile) ReadAt(p []byte, off int64) (int, error) {
if off >= int64(len(f.n.mu.data)) {
return 0, io.EOF
}
return copy(p, f.n.mu.data[off:]), nil
n := copy(p, f.n.mu.data[off:])
if n < len(p) {
return n, io.EOF
}
return n, nil
}
func (f *memFile) Write(p []byte) (int, error) {
@@ -698,6 +743,29 @@ func (f *memFile) Write(p []byte) (int, error) {
return len(p), nil
}
func (f *memFile) WriteAt(p []byte, ofs int64) (int, error) {
if !f.write {
return 0, errors.New("pebble/vfs: file was not created for writing")
}
if f.n.isDir {
return 0, errors.New("pebble/vfs: cannot write a directory")
}
f.n.mu.Lock()
defer f.n.mu.Unlock()
f.n.mu.modTime = time.Now()
for len(f.n.mu.data) < int(ofs)+len(p) {
f.n.mu.data = append(f.n.mu.data, 0)
}
n := copy(f.n.mu.data[int(ofs):int(ofs)+len(p)], p)
if n != len(p) {
panic("stuff")
}
return len(p), nil
}
func (f *memFile) Prefetch(offset int64, length int64) error { return nil }
func (f *memFile) Preallocate(offset, length int64) error { return nil }
@@ -747,3 +815,18 @@ func (f *memFile) Fd() uintptr {
func (f *memFile) Flush() error {
return nil
}
type memFileLock struct {
y *MemFS
f File
fullname string
}
func (l *memFileLock) Close() error {
if l.y == nil {
return nil
}
l.y.lockedFiles.Delete(l.fullname)
l.y = nil
return l.f.Close()
}

36
vendor/github.com/cockroachdb/pebble/vfs/syncing_file.go generated vendored
View File

@@ -26,16 +26,14 @@ type syncingFile struct {
noSyncOnClose bool
bytesPerSync int64
preallocateSize int64
atomic struct {
// The offset at which dirty data has been written.
offset int64
// The offset at which data has been synced. Note that if SyncFileRange is
// being used, the periodic syncing of data during writing will only ever
// sync up to offset-1MB. This is done to avoid rewriting the tail of the
// file multiple times, but has the side effect of ensuring that Close will
// sync the file's metadata.
syncOffset int64
}
// The offset at which dirty data has been written.
offset atomic.Int64
// The offset at which data has been synced. Note that if SyncFileRange is
// being used, the periodic syncing of data during writing will only ever
// sync up to offset-1MB. This is done to avoid rewriting the tail of the
// file multiple times, but has the side effect of ensuring that Close will
// sync the file's metadata.
syncOffset atomic.Int64
preallocatedBlocks int64
}
@@ -54,13 +52,13 @@ func NewSyncingFile(f File, opts SyncingFileOptions) File {
}
// Ensure a file that is opened and then closed will be synced, even if no
// data has been written to it.
s.atomic.syncOffset = -1
s.syncOffset.Store(-1)
return s
}
// NB: syncingFile.Write is unsafe for concurrent use!
func (f *syncingFile) Write(p []byte) (n int, err error) {
_ = f.preallocate(atomic.LoadInt64(&f.atomic.offset))
_ = f.preallocate(f.offset.Load())
n, err = f.File.Write(p)
if err != nil {
@@ -68,7 +66,7 @@ func (f *syncingFile) Write(p []byte) (n int, err error) {
}
// The offset is updated atomically so that it can be accessed safely from
// Sync.
atomic.AddInt64(&f.atomic.offset, int64(n))
f.offset.Add(int64(n))
if err := f.maybeSync(); err != nil {
return 0, err
}
@@ -93,11 +91,11 @@ func (f *syncingFile) preallocate(offset int64) error {
func (f *syncingFile) ratchetSyncOffset(offset int64) {
for {
syncOffset := atomic.LoadInt64(&f.atomic.syncOffset)
syncOffset := f.syncOffset.Load()
if syncOffset >= offset {
return
}
if atomic.CompareAndSwapInt64(&f.atomic.syncOffset, syncOffset, offset) {
if f.syncOffset.CompareAndSwap(syncOffset, offset) {
return
}
}
@@ -109,7 +107,7 @@ func (f *syncingFile) Sync() error {
// offset, we still need to call the underlying file's sync for persistence
// guarantees which are not provided by SyncTo (or by sync_file_range on
// Linux).
f.ratchetSyncOffset(atomic.LoadInt64(&f.atomic.offset))
f.ratchetSyncOffset(f.offset.Load())
return f.SyncData()
}
@@ -127,7 +125,7 @@ func (f *syncingFile) maybeSync() error {
// Xfs does neighbor page flushing outside of the specified ranges. We
// need to make sure sync range is far from the write offset.
const syncRangeBuffer = 1 << 20 // 1 MB
offset := atomic.LoadInt64(&f.atomic.offset)
offset := f.offset.Load()
if offset <= syncRangeBuffer {
return nil
}
@@ -135,7 +133,7 @@ func (f *syncingFile) maybeSync() error {
const syncRangeAlignment = 4 << 10 // 4 KB
syncToOffset := offset - syncRangeBuffer
syncToOffset -= syncToOffset % syncRangeAlignment
syncOffset := atomic.LoadInt64(&f.atomic.syncOffset)
syncOffset := f.syncOffset.Load()
if syncToOffset < 0 || (syncToOffset-syncOffset) < f.bytesPerSync {
return nil
}
@@ -167,7 +165,7 @@ func (f *syncingFile) Close() error {
// caller has not called Sync since the last write, syncOffset is guaranteed
// to be less than atomic.offset. This ensures we fall into the below
// conditional and perform a full sync to durably persist the file.
if off := atomic.LoadInt64(&f.atomic.offset); off > atomic.LoadInt64(&f.atomic.syncOffset) {
if off := f.offset.Load(); off > f.syncOffset.Load() {
// There's still remaining dirty data.
if f.noSyncOnClose {

18
vendor/github.com/cockroachdb/pebble/vfs/vfs.go generated vendored
View File

@@ -29,6 +29,8 @@ type File interface {
// method *is* allowed to modify the slice passed in, whether temporarily
// or permanently. Callers of Write() need to take this into account.
io.Writer
// WriteAt() is only supported for files that were opened with FS.OpenReadWrite.
io.WriterAt
// Preallocate optionally preallocates storage for `length` at `offset`
// within the file. Implementations may choose to do nothing.
@@ -99,6 +101,10 @@ type FS interface {
// Open opens the named file for reading. openOptions provides
Open(name string, opts ...OpenOption) (File, error)
// OpenReadWrite opens the named file for reading and writing. If the file
// does not exist, it is created.
OpenReadWrite(name string, opts ...OpenOption) (File, error)
// OpenDir opens the named directory for syncing.
OpenDir(name string) (File, error)
@@ -232,6 +238,18 @@ func (defaultFS) Open(name string, opts ...OpenOption) (File, error) {
return file, nil
}
func (defaultFS) OpenReadWrite(name string, opts ...OpenOption) (File, error) {
osFile, err := os.OpenFile(name, os.O_RDWR|syscall.O_CLOEXEC|os.O_CREATE, 0666)
if err != nil {
return nil, errors.WithStack(err)
}
file := wrapOSFile(osFile)
for _, opt := range opts {
opt.Apply(file)
}
return file, nil
}
func (defaultFS) Remove(name string) error {
return errors.WithStack(os.Remove(name))
}