Initial commit

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing.go

@@ -0,0 +1,68 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objiotracing
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// OpType indicates the type of operation.
+type OpType uint8
+
+// OpType values.
+const (
+	ReadOp OpType = iota
+	WriteOp
+	// RecordCacheHitOp happens when a read is satisfied from the block cache. See
+	// objstorage.ReadHandle.RecordCacheHit().
+	RecordCacheHitOp
+	// SetupForCompactionOp is a "meta operation" that configures a read handle
+	// for large sequential reads. See objstorage.ReadHandle.SetupForCompaction().
+	SetupForCompactionOp
+)
+
+// Reason indicates the higher-level context of the operation.
+type Reason uint8
+
+// Reason values.
+const (
+	UnknownReason Reason = iota
+	ForFlush
+	ForCompaction
+	ForIngestion
+	// TODO(radu): add ForUserFacing.
+)
+
+// BlockType indicates the type of data block relevant to an operation.
+type BlockType uint8
+
+// BlockType values.
+const (
+	UnknownBlock BlockType = iota
+	DataBlock
+	ValueBlock
+	FilterBlock
+	MetadataBlock
+)
+
+// Event is the on-disk format of a tracing event. It is exported here so that
+// trace processing tools can use it by importing this package.
+type Event struct {
+	// Event start time as a Unix time (see time.Time.StartUnixNano()).
+	// Note that recorded events are not necessarily ordered by time - this is
+	// because separate event "streams" use local buffers (for performance).
+	StartUnixNano int64
+	Op            OpType
+	Reason        Reason
+	BlockType     BlockType
+	// LSM level plus one (with 0 indicating unknown level).
+	LevelPlusOne uint8
+	// Hardcoded padding so that struct layout doesn't depend on architecture.
+	_       uint32
+	FileNum base.FileNum
+	// HandleID is a unique identifier corresponding to an objstorage.ReadHandle;
+	// only set for read operations performed through a ReadHandle.
+	HandleID uint64
+	Offset   int64
+	Size     int64
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go

@@ -0,0 +1,59 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build !pebble_obj_io_tracing
+// +build !pebble_obj_io_tracing
+
+package objiotracing
+
+import (
+	"context"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// Enabled is used to short circuit tracing-related code in regular builds.
+const Enabled = false
+
+// Tracer manages the writing of object IO traces to files.
+type Tracer struct{}
+
+// Open creates a Tracer which generates trace files in the given directory.
+// Each trace file contains a series of Events (as they are in memory).
+func Open(fs vfs.FS, fsDir string) *Tracer {
+	return nil
+}
+
+// Close the tracer, flushing any remaining events.
+func (*Tracer) Close() {}
+
+// WrapReadable wraps an objstorage.Readable with one that generates tracing
+// events.
+func (*Tracer) WrapReadable(
+	ctx context.Context, r objstorage.Readable, fileNum base.DiskFileNum,
+) objstorage.Readable {
+	return r
+}
+
+// WrapWritable wraps an objstorage.Writable with one that generates tracing
+// events.
+func (t *Tracer) WrapWritable(
+	ctx context.Context, w objstorage.Writable, fileNum base.DiskFileNum,
+) objstorage.Writable {
+	return w
+}
+
+// WithReason creates a context that has an associated Reason (which ends up in
+// traces created under that context).
+func WithReason(ctx context.Context, reason Reason) context.Context { return ctx }
+
+// WithBlockType creates a context that has an associated BlockType (which ends up in
+// traces created under that context).
+func WithBlockType(ctx context.Context, blockType BlockType) context.Context { return ctx }
+
+// WithLevel creates a context that has an associated level (which ends up in
+// traces created under that context).
+func WithLevel(ctx context.Context, level int) context.Context { return ctx }

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go

@@ -0,0 +1,410 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build pebble_obj_io_tracing
+// +build pebble_obj_io_tracing
+
+package objiotracing
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+	"time"
+	"unsafe"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// Enabled is used to short circuit tracing-related code in regular builds.
+const Enabled = true
+
+// Tracer manages the writing of object IO traces to files.
+//
+// The tracer runs a background worker goroutine which receives trace event
+// buffers over a channel and dumps them to IOTRACES- files. Wrapper
+// implementations for Readable, ReadHandle, Writable are producers of traces;
+// they maintain internal buffers of events which get flushed to the buffered
+// channel when they get full. This allows for minimal synchronization per IO
+// (as for most of these structures, an instance only allows a single IO at a
+// time).
+type Tracer struct {
+	fs    vfs.FS
+	fsDir string
+
+	handleID atomic.Uint64
+
+	workerStopCh chan struct{}
+	workerDataCh chan eventBuf
+	workerWait   sync.WaitGroup
+}
+
+// Open creates a Tracer which generates trace files in the given directory.
+// Each trace file contains a series of Events (as they are in memory).
+func Open(fs vfs.FS, fsDir string) *Tracer {
+	t := &Tracer{
+		fs:           fs,
+		fsDir:        fsDir,
+		workerStopCh: make(chan struct{}),
+		workerDataCh: make(chan eventBuf, channelBufSize),
+	}
+
+	t.handleID.Store(uint64(rand.NewSource(time.Now().UnixNano()).Int63()))
+
+	t.workerWait.Add(1)
+	go t.workerLoop()
+	return t
+}
+
+// Close the tracer, flushing any remaining events.
+func (t *Tracer) Close() {
+	if t.workerStopCh == nil {
+		return
+	}
+	// Tell the worker to stop and wait for it to finish up.
+	close(t.workerStopCh)
+	t.workerWait.Wait()
+	t.workerStopCh = nil
+}
+
+// WrapWritable wraps an objstorage.Writable with one that generates tracing
+// events.
+func (t *Tracer) WrapWritable(
+	ctx context.Context, w objstorage.Writable, fileNum base.FileNum,
+) objstorage.Writable {
+	return &writable{
+		w:       w,
+		fileNum: fileNum,
+		g:       makeEventGenerator(ctx, t),
+	}
+}
+
+type writable struct {
+	w         objstorage.Writable
+	fileNum   base.FileNum
+	curOffset int64
+	g         eventGenerator
+}
+
+var _ objstorage.Writable = (*writable)(nil)
+
+// Write is part of the objstorage.Writable interface.
+func (w *writable) Write(p []byte) error {
+	w.g.add(context.Background(), Event{
+		Op:      WriteOp,
+		FileNum: w.fileNum,
+		Offset:  w.curOffset,
+		Size:    int64(len(p)),
+	})
+	// If w.w.Write(p) returns an error, a new writable
+	// will be used, so even tho all of p may not have
+	// been written to the underlying "file", it is okay
+	// to add len(p) to curOffset.
+	w.curOffset += int64(len(p))
+	return w.w.Write(p)
+}
+
+// Finish is part of the objstorage.Writable interface.
+func (w *writable) Finish() error {
+	w.g.flush()
+	return w.w.Finish()
+}
+
+// Abort is part of the objstorage.Writable interface.
+func (w *writable) Abort() {
+	w.g.flush()
+	w.w.Abort()
+}
+
+// WrapReadable wraps an objstorage.Readable with one that generates tracing
+// events.
+func (t *Tracer) WrapReadable(
+	ctx context.Context, r objstorage.Readable, fileNum base.FileNum,
+) objstorage.Readable {
+	res := &readable{
+		r:       r,
+		fileNum: fileNum,
+	}
+	res.mu.g = makeEventGenerator(ctx, t)
+	return res
+}
+
+type readable struct {
+	r       objstorage.Readable
+	fileNum base.FileNum
+	mu      struct {
+		sync.Mutex
+		g eventGenerator
+	}
+}
+
+var _ objstorage.Readable = (*readable)(nil)
+
+// ReadAt is part of the objstorage.Readable interface.
+func (r *readable) ReadAt(ctx context.Context, v []byte, off int64) (n int, err error) {
+	r.mu.Lock()
+	r.mu.g.add(ctx, Event{
+		Op:      ReadOp,
+		FileNum: r.fileNum,
+		Offset:  off,
+		Size:    int64(len(v)),
+	})
+	r.mu.Unlock()
+	return r.r.ReadAt(ctx, v, off)
+}
+
+// Close is part of the objstorage.Readable interface.
+func (r *readable) Close() error {
+	r.mu.g.flush()
+	return r.r.Close()
+}
+
+// Size is part of the objstorage.Readable interface.
+func (r *readable) Size() int64 {
+	return r.r.Size()
+}
+
+// NewReadHandle is part of the objstorage.Readable interface.
+func (r *readable) NewReadHandle(ctx context.Context) objstorage.ReadHandle {
+	// It's safe to get the tracer from the generator without the mutex since it never changes.
+	t := r.mu.g.t
+	return &readHandle{
+		rh:       r.r.NewReadHandle(ctx),
+		fileNum:  r.fileNum,
+		handleID: t.handleID.Add(1),
+		g:        makeEventGenerator(ctx, t),
+	}
+}
+
+type readHandle struct {
+	rh       objstorage.ReadHandle
+	fileNum  base.FileNum
+	handleID uint64
+	g        eventGenerator
+}
+
+var _ objstorage.ReadHandle = (*readHandle)(nil)
+
+// ReadAt is part of the objstorage.ReadHandle interface.
+func (rh *readHandle) ReadAt(ctx context.Context, p []byte, off int64) (n int, err error) {
+	rh.g.add(ctx, Event{
+		Op:       ReadOp,
+		FileNum:  rh.fileNum,
+		HandleID: rh.handleID,
+		Offset:   off,
+		Size:     int64(len(p)),
+	})
+	return rh.rh.ReadAt(ctx, p, off)
+}
+
+// Close is part of the objstorage.ReadHandle interface.
+func (rh *readHandle) Close() error {
+	rh.g.flush()
+	return rh.rh.Close()
+}
+
+// SetupForCompaction is part of the objstorage.ReadHandle interface.
+func (rh *readHandle) SetupForCompaction() {
+	rh.g.add(context.Background(), Event{
+		Op:       SetupForCompactionOp,
+		FileNum:  rh.fileNum,
+		HandleID: rh.handleID,
+	})
+	rh.rh.SetupForCompaction()
+}
+
+// RecordCacheHit is part of the objstorage.ReadHandle interface.
+func (rh *readHandle) RecordCacheHit(ctx context.Context, offset, size int64) {
+	rh.g.add(ctx, Event{
+		Op:       RecordCacheHitOp,
+		FileNum:  rh.fileNum,
+		HandleID: rh.handleID,
+		Offset:   offset,
+		Size:     size,
+	})
+	rh.rh.RecordCacheHit(ctx, offset, size)
+}
+
+type ctxInfo struct {
+	reason       Reason
+	blockType    BlockType
+	levelPlusOne uint8
+}
+
+func mergeCtxInfo(base, other ctxInfo) ctxInfo {
+	res := other
+	if res.reason == 0 {
+		res.reason = base.reason
+	}
+	if res.blockType == 0 {
+		res.blockType = base.blockType
+	}
+	if res.levelPlusOne == 0 {
+		res.levelPlusOne = base.levelPlusOne
+	}
+	return res
+}
+
+type ctxInfoKey struct{}
+
+func withInfo(ctx context.Context, info ctxInfo) context.Context {
+	return context.WithValue(ctx, ctxInfoKey{}, info)
+}
+
+func infoFromCtx(ctx context.Context) ctxInfo {
+	res := ctx.Value(ctxInfoKey{})
+	if res == nil {
+		return ctxInfo{}
+	}
+	return res.(ctxInfo)
+}
+
+// WithReason creates a context that has an associated Reason (which ends up in
+// traces created under that context).
+func WithReason(ctx context.Context, reason Reason) context.Context {
+	info := infoFromCtx(ctx)
+	info.reason = reason
+	return withInfo(ctx, info)
+}
+
+// WithBlockType creates a context that has an associated BlockType (which ends up in
+// traces created under that context).
+func WithBlockType(ctx context.Context, blockType BlockType) context.Context {
+	info := infoFromCtx(ctx)
+	info.blockType = blockType
+	return withInfo(ctx, info)
+}
+
+// WithLevel creates a context that has an associated level (which ends up in
+// traces created under that context).
+func WithLevel(ctx context.Context, level int) context.Context {
+	info := infoFromCtx(ctx)
+	info.levelPlusOne = uint8(level) + 1
+	return withInfo(ctx, info)
+}
+
+const (
+	eventSize            = int(unsafe.Sizeof(Event{}))
+	targetEntriesPerFile = 256 * 1024 * 1024 / eventSize // 256MB files
+	eventsPerBuf         = 16
+	channelBufSize       = 512 * 1024 / eventsPerBuf // 512K events.
+	bytesPerFileSync     = 128 * 1024
+)
+
+type eventBuf struct {
+	events [eventsPerBuf]Event
+	num    int
+}
+
+type eventGenerator struct {
+	t           *Tracer
+	baseCtxInfo ctxInfo
+	buf         eventBuf
+}
+
+func makeEventGenerator(ctx context.Context, t *Tracer) eventGenerator {
+	return eventGenerator{
+		t:           t,
+		baseCtxInfo: infoFromCtx(ctx),
+	}
+}
+
+func (g *eventGenerator) flush() {
+	if g.buf.num > 0 {
+		g.t.workerDataCh <- g.buf
+		g.buf.num = 0
+	}
+}
+
+func (g *eventGenerator) add(ctx context.Context, e Event) {
+	e.StartUnixNano = time.Now().UnixNano()
+	info := infoFromCtx(ctx)
+	info = mergeCtxInfo(g.baseCtxInfo, info)
+	e.Reason = info.reason
+	e.BlockType = info.blockType
+	e.LevelPlusOne = info.levelPlusOne
+	if g.buf.num == eventsPerBuf {
+		g.flush()
+	}
+	g.buf.events[g.buf.num] = e
+	g.buf.num++
+}
+
+type workerState struct {
+	curFile          vfs.File
+	curBW            *bufio.Writer
+	numEntriesInFile int
+}
+
+func (t *Tracer) workerLoop() {
+	defer t.workerWait.Done()
+	stopCh := t.workerStopCh
+	dataCh := t.workerDataCh
+	var state workerState
+	t.workerNewFile(&state)
+	for {
+		select {
+		case <-stopCh:
+			close(dataCh)
+			// Flush any remaining traces.
+			for data := range dataCh {
+				t.workerWriteTraces(&state, data)
+			}
+			t.workerCloseFile(&state)
+			return
+
+		case data := <-dataCh:
+			t.workerWriteTraces(&state, data)
+		}
+	}
+}
+
+func (t *Tracer) workerWriteTraces(state *workerState, data eventBuf) {
+	if state.numEntriesInFile >= targetEntriesPerFile {
+		t.workerCloseFile(state)
+		t.workerNewFile(state)
+	}
+	state.numEntriesInFile += data.num
+	p := unsafe.Pointer(&data.events[0])
+	b := unsafe.Slice((*byte)(p), eventSize*data.num)
+	if _, err := state.curBW.Write(b); err != nil {
+		panic(err)
+	}
+}
+
+func (t *Tracer) workerNewFile(state *workerState) {
+	filename := fmt.Sprintf("IOTRACES-%s", time.Now().UTC().Format(time.RFC3339Nano))
+
+	file, err := t.fs.Create(t.fs.PathJoin(t.fsDir, filename))
+	if err != nil {
+		panic(err)
+	}
+	file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{
+		BytesPerSync: bytesPerFileSync,
+	})
+	state.curFile = file
+	state.curBW = bufio.NewWriter(file)
+	state.numEntriesInFile = 0
+}
+
+func (t *Tracer) workerCloseFile(state *workerState) {
+	if state.curFile != nil {
+		if err := state.curBW.Flush(); err != nil {
+			panic(err)
+		}
+		if err := state.curFile.Sync(); err != nil {
+			panic(err)
+		}
+		if err := state.curFile.Close(); err != nil {
+			panic(err)
+		}
+		state.curFile = nil
+		state.curBW = nil
+	}
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go

@@ -0,0 +1,570 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"io"
+	"os"
+	"sort"
+	"sync"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/errors/oserror"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// provider is the implementation of objstorage.Provider.
+type provider struct {
+	st Settings
+
+	fsDir vfs.File
+
+	tracer *objiotracing.Tracer
+
+	remote remoteSubsystem
+
+	mu struct {
+		sync.RWMutex
+
+		remote struct {
+			// catalogBatch accumulates remote object creations and deletions until
+			// Sync is called.
+			catalogBatch remoteobjcat.Batch
+
+			storageObjects map[remote.Locator]remote.Storage
+		}
+
+		// localObjectsChanged is set if non-remote objects were created or deleted
+		// but Sync was not yet called.
+		localObjectsChanged bool
+
+		// knownObjects maintains information about objects that are known to the provider.
+		// It is initialized with the list of files in the manifest when we open a DB.
+		knownObjects map[base.DiskFileNum]objstorage.ObjectMetadata
+
+		// protectedObjects are objects that cannot be unreferenced because they
+		// have outstanding SharedObjectBackingHandles. The value is a count of outstanding handles
+		protectedObjects map[base.DiskFileNum]int
+	}
+}
+
+var _ objstorage.Provider = (*provider)(nil)
+
+// Settings that must be specified when creating the provider.
+type Settings struct {
+	Logger base.Logger
+
+	// Local filesystem configuration.
+	FS        vfs.FS
+	FSDirName string
+
+	// FSDirInitialListing is a listing of FSDirName at the time of calling Open.
+	//
+	// This is an optional optimization to avoid double listing on Open when the
+	// higher layer already has a listing. When nil, we obtain the listing on
+	// Open.
+	FSDirInitialListing []string
+
+	// Cleaner cleans obsolete files from the local filesystem.
+	//
+	// The default cleaner uses the DeleteCleaner.
+	FSCleaner base.Cleaner
+
+	// NoSyncOnClose decides whether the implementation will enforce a
+	// close-time synchronization (e.g., fdatasync() or sync_file_range())
+	// on files it writes to. Setting this to true removes the guarantee for a
+	// sync on close. Some implementations can still issue a non-blocking sync.
+	NoSyncOnClose bool
+
+	// BytesPerSync enables periodic syncing of files in order to smooth out
+	// writes to disk. This option does not provide any persistence guarantee, but
+	// is used to avoid latency spikes if the OS automatically decides to write
+	// out a large chunk of dirty filesystem buffers.
+	BytesPerSync int
+
+	// Local contains fields that are only relevant for files stored on the local
+	// filesystem.
+	Local struct {
+		// TODO(radu): move FSCleaner, NoSyncOnClose, BytesPerSync here.
+
+		// ReadaheadConfigFn is a function used to retrieve the current readahead
+		// mode. This function is run whenever a local object is open for reading.
+		// If it is nil, DefaultReadaheadConfig is used.
+		ReadaheadConfigFn func() ReadaheadConfig
+	}
+
+	// Fields here are set only if the provider is to support remote objects
+	// (experimental).
+	Remote struct {
+		StorageFactory remote.StorageFactory
+
+		// If CreateOnShared is non-zero, sstables are created on remote storage using
+		// the CreateOnSharedLocator (when the PreferSharedStorage create option is
+		// true).
+		CreateOnShared        remote.CreateOnSharedStrategy
+		CreateOnSharedLocator remote.Locator
+
+		// CacheSizeBytes is the size of the on-disk block cache for objects
+		// on remote storage. If it is 0, no cache is used.
+		CacheSizeBytes int64
+
+		// CacheBlockSize is the block size of the cache; if 0, the default of 32KB is used.
+		CacheBlockSize int
+
+		// ShardingBlockSize is the size of a shard block. The cache is split into contiguous
+		// ShardingBlockSize units. The units are distributed across multiple independent shards
+		// of the cache, via a hash(offset) modulo num shards operation. The cache replacement
+		// policies operate at the level of shard, not whole cache. This is done to reduce lock
+		// contention.
+		//
+		// If ShardingBlockSize is 0, the default of 1 MB is used.
+		ShardingBlockSize int64
+
+		// The number of independent shards the cache leverages. Each shard is the same size,
+		// and a hash of filenum & offset map a read to a certain shard. If set to 0,
+		// 2*runtime.GOMAXPROCS is used as the shard count.
+		CacheShardCount int
+
+		// TODO(radu): allow the cache to live on another FS/location (e.g. to use
+		// instance-local SSD).
+	}
+}
+
+// ReadaheadConfig controls the use of read-ahead.
+type ReadaheadConfig struct {
+	// Informed is the type of read-ahead for operations that are known to read a
+	// large consecutive chunk of a file.
+	Informed ReadaheadMode
+
+	// Speculative is the type of read-ahead used automatically, when consecutive
+	// reads are detected.
+	Speculative ReadaheadMode
+}
+
+// DefaultReadaheadConfig is the readahead config used when ReadaheadConfigFn is
+// not specified.
+var DefaultReadaheadConfig = ReadaheadConfig{
+	Informed:    FadviseSequential,
+	Speculative: FadviseSequential,
+}
+
+// ReadaheadMode indicates the type of read-ahead to use, either for informed
+// read-ahead (e.g. compactions) or speculative read-ahead.
+type ReadaheadMode uint8
+
+const (
+	// NoReadahead disables readahead altogether.
+	NoReadahead ReadaheadMode = iota
+
+	// SysReadahead enables the use of SYS_READAHEAD call to prefetch data.
+	// The prefetch window grows dynamically as consecutive writes are detected.
+	SysReadahead
+
+	// FadviseSequential enables to use of FADV_SEQUENTIAL. For informed
+	// read-ahead, FADV_SEQUENTIAL is used from the beginning. For speculative
+	// read-ahead SYS_READAHEAD is first used until the window reaches the maximum
+	// size, then we siwtch to FADV_SEQUENTIAL.
+	FadviseSequential
+)
+
+// DefaultSettings initializes default settings (with no remote storage),
+// suitable for tests and tools.
+func DefaultSettings(fs vfs.FS, dirName string) Settings {
+	return Settings{
+		Logger:        base.DefaultLogger,
+		FS:            fs,
+		FSDirName:     dirName,
+		FSCleaner:     base.DeleteCleaner{},
+		NoSyncOnClose: false,
+		BytesPerSync:  512 * 1024, // 512KB
+	}
+}
+
+// Open creates the provider.
+func Open(settings Settings) (objstorage.Provider, error) {
+	// Note: we can't just `return open(settings)` because in an error case we
+	// would return (*provider)(nil) which is not objstorage.Provider(nil).
+	p, err := open(settings)
+	if err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+func open(settings Settings) (p *provider, _ error) {
+	fsDir, err := settings.FS.OpenDir(settings.FSDirName)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if p == nil {
+			fsDir.Close()
+		}
+	}()
+
+	p = &provider{
+		st:    settings,
+		fsDir: fsDir,
+	}
+	p.mu.knownObjects = make(map[base.DiskFileNum]objstorage.ObjectMetadata)
+	p.mu.protectedObjects = make(map[base.DiskFileNum]int)
+
+	if objiotracing.Enabled {
+		p.tracer = objiotracing.Open(settings.FS, settings.FSDirName)
+	}
+
+	// Add local FS objects.
+	if err := p.vfsInit(); err != nil {
+		return nil, err
+	}
+
+	// Initialize remote subsystem (if configured) and add remote objects.
+	if err := p.remoteInit(); err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}
+
+// Close is part of the objstorage.Provider interface.
+func (p *provider) Close() error {
+	err := p.sharedClose()
+	if p.fsDir != nil {
+		err = firstError(err, p.fsDir.Close())
+		p.fsDir = nil
+	}
+	if objiotracing.Enabled {
+		if p.tracer != nil {
+			p.tracer.Close()
+			p.tracer = nil
+		}
+	}
+	return err
+}
+
+// OpenForReading opens an existing object.
+func (p *provider) OpenForReading(
+	ctx context.Context,
+	fileType base.FileType,
+	fileNum base.DiskFileNum,
+	opts objstorage.OpenOptions,
+) (objstorage.Readable, error) {
+	meta, err := p.Lookup(fileType, fileNum)
+	if err != nil {
+		if opts.MustExist {
+			p.st.Logger.Fatalf("%v", err)
+		}
+		return nil, err
+	}
+
+	var r objstorage.Readable
+	if !meta.IsRemote() {
+		r, err = p.vfsOpenForReading(ctx, fileType, fileNum, opts)
+	} else {
+		r, err = p.remoteOpenForReading(ctx, meta, opts)
+		if err != nil && p.isNotExistError(meta, err) {
+			// Wrap the error so that IsNotExistError functions properly.
+			err = errors.Mark(err, os.ErrNotExist)
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+	if objiotracing.Enabled {
+		r = p.tracer.WrapReadable(ctx, r, fileNum)
+	}
+	return r, nil
+}
+
+// Create creates a new object and opens it for writing.
+//
+// The object is not guaranteed to be durable (accessible in case of crashes)
+// until Sync is called.
+func (p *provider) Create(
+	ctx context.Context,
+	fileType base.FileType,
+	fileNum base.DiskFileNum,
+	opts objstorage.CreateOptions,
+) (w objstorage.Writable, meta objstorage.ObjectMetadata, err error) {
+	if opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone {
+		w, meta, err = p.sharedCreate(ctx, fileType, fileNum, p.st.Remote.CreateOnSharedLocator, opts)
+	} else {
+		w, meta, err = p.vfsCreate(ctx, fileType, fileNum)
+	}
+	if err != nil {
+		err = errors.Wrapf(err, "creating object %s", errors.Safe(fileNum))
+		return nil, objstorage.ObjectMetadata{}, err
+	}
+	p.addMetadata(meta)
+	if objiotracing.Enabled {
+		w = p.tracer.WrapWritable(ctx, w, fileNum)
+	}
+	return w, meta, nil
+}
+
+// Remove removes an object.
+//
+// Note that if the object is remote, the object is only (conceptually) removed
+// from this provider. If other providers have references on the remote object,
+// it will not be removed.
+//
+// The object is not guaranteed to be durably removed until Sync is called.
+func (p *provider) Remove(fileType base.FileType, fileNum base.DiskFileNum) error {
+	meta, err := p.Lookup(fileType, fileNum)
+	if err != nil {
+		return err
+	}
+
+	if !meta.IsRemote() {
+		err = p.vfsRemove(fileType, fileNum)
+	} else {
+		// TODO(radu): implement remote object removal (i.e. deref).
+		err = p.sharedUnref(meta)
+		if err != nil && p.isNotExistError(meta, err) {
+			// Wrap the error so that IsNotExistError functions properly.
+			err = errors.Mark(err, os.ErrNotExist)
+		}
+	}
+	if err != nil && !p.IsNotExistError(err) {
+		// We want to be able to retry a Remove, so we keep the object in our list.
+		// TODO(radu): we should mark the object as "zombie" and not allow any other
+		// operations.
+		return errors.Wrapf(err, "removing object %s", errors.Safe(fileNum))
+	}
+
+	p.removeMetadata(fileNum)
+	return err
+}
+
+func (p *provider) isNotExistError(meta objstorage.ObjectMetadata, err error) bool {
+	if meta.Remote.Storage != nil {
+		return meta.Remote.Storage.IsNotExistError(err)
+	}
+	return oserror.IsNotExist(err)
+}
+
+// IsNotExistError is part of the objstorage.Provider interface.
+func (p *provider) IsNotExistError(err error) bool {
+	// We use errors.Mark(err, os.ErrNotExist) for not-exist errors coming from
+	// remote.Storage.
+	return oserror.IsNotExist(err)
+}
+
+// Sync flushes the metadata from creation or removal of objects since the last Sync.
+func (p *provider) Sync() error {
+	if err := p.vfsSync(); err != nil {
+		return err
+	}
+	if err := p.sharedSync(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// LinkOrCopyFromLocal creates a new object that is either a copy of a given
+// local file or a hard link (if the new object is created on the same FS, and
+// if the FS supports it).
+//
+// The object is not guaranteed to be durable (accessible in case of crashes)
+// until Sync is called.
+func (p *provider) LinkOrCopyFromLocal(
+	ctx context.Context,
+	srcFS vfs.FS,
+	srcFilePath string,
+	dstFileType base.FileType,
+	dstFileNum base.DiskFileNum,
+	opts objstorage.CreateOptions,
+) (objstorage.ObjectMetadata, error) {
+	shared := opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone
+	if !shared && srcFS == p.st.FS {
+		// Wrap the normal filesystem with one which wraps newly created files with
+		// vfs.NewSyncingFile.
+		fs := vfs.NewSyncingFS(p.st.FS, vfs.SyncingFileOptions{
+			NoSyncOnClose: p.st.NoSyncOnClose,
+			BytesPerSync:  p.st.BytesPerSync,
+		})
+		dstPath := p.vfsPath(dstFileType, dstFileNum)
+		if err := vfs.LinkOrCopy(fs, srcFilePath, dstPath); err != nil {
+			return objstorage.ObjectMetadata{}, err
+		}
+
+		meta := objstorage.ObjectMetadata{
+			DiskFileNum: dstFileNum,
+			FileType:    dstFileType,
+		}
+		p.addMetadata(meta)
+		return meta, nil
+	}
+	// Create the object and copy the data.
+	w, meta, err := p.Create(ctx, dstFileType, dstFileNum, opts)
+	if err != nil {
+		return objstorage.ObjectMetadata{}, err
+	}
+	f, err := srcFS.Open(srcFilePath, vfs.SequentialReadsOption)
+	if err != nil {
+		return objstorage.ObjectMetadata{}, err
+	}
+	defer f.Close()
+	buf := make([]byte, 64*1024)
+	for {
+		n, readErr := f.Read(buf)
+		if readErr != nil && readErr != io.EOF {
+			w.Abort()
+			return objstorage.ObjectMetadata{}, readErr
+		}
+
+		if n > 0 {
+			if err := w.Write(buf[:n]); err != nil {
+				w.Abort()
+				return objstorage.ObjectMetadata{}, err
+			}
+		}
+
+		if readErr == io.EOF {
+			break
+		}
+	}
+	if err := w.Finish(); err != nil {
+		return objstorage.ObjectMetadata{}, err
+	}
+	return meta, nil
+}
+
+// Lookup is part of the objstorage.Provider interface.
+func (p *provider) Lookup(
+	fileType base.FileType, fileNum base.DiskFileNum,
+) (objstorage.ObjectMetadata, error) {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	meta, ok := p.mu.knownObjects[fileNum]
+	if !ok {
+		return objstorage.ObjectMetadata{}, errors.Wrapf(
+			os.ErrNotExist,
+			"file %s (type %d) unknown to the objstorage provider",
+			errors.Safe(fileNum), errors.Safe(fileType),
+		)
+	}
+	if meta.FileType != fileType {
+		return objstorage.ObjectMetadata{}, errors.AssertionFailedf(
+			"file %s type mismatch (known type %d, expected type %d)",
+			errors.Safe(fileNum), errors.Safe(meta.FileType), errors.Safe(fileType),
+		)
+	}
+	return meta, nil
+}
+
+// Path is part of the objstorage.Provider interface.
+func (p *provider) Path(meta objstorage.ObjectMetadata) string {
+	if !meta.IsRemote() {
+		return p.vfsPath(meta.FileType, meta.DiskFileNum)
+	}
+	return p.remotePath(meta)
+}
+
+// Size returns the size of the object.
+func (p *provider) Size(meta objstorage.ObjectMetadata) (int64, error) {
+	if !meta.IsRemote() {
+		return p.vfsSize(meta.FileType, meta.DiskFileNum)
+	}
+	return p.remoteSize(meta)
+}
+
+// List is part of the objstorage.Provider interface.
+func (p *provider) List() []objstorage.ObjectMetadata {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	res := make([]objstorage.ObjectMetadata, 0, len(p.mu.knownObjects))
+	for _, meta := range p.mu.knownObjects {
+		res = append(res, meta)
+	}
+	sort.Slice(res, func(i, j int) bool {
+		return res[i].DiskFileNum.FileNum() < res[j].DiskFileNum.FileNum()
+	})
+	return res
+}
+
+// Metrics is part of the objstorage.Provider interface.
+func (p *provider) Metrics() sharedcache.Metrics {
+	if p.remote.cache != nil {
+		return p.remote.cache.Metrics()
+	}
+	return sharedcache.Metrics{}
+}
+
+func (p *provider) addMetadata(meta objstorage.ObjectMetadata) {
+	if invariants.Enabled {
+		meta.AssertValid()
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.mu.knownObjects[meta.DiskFileNum] = meta
+	if meta.IsRemote() {
+		p.mu.remote.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
+			FileNum:        meta.DiskFileNum,
+			FileType:       meta.FileType,
+			CreatorID:      meta.Remote.CreatorID,
+			CreatorFileNum: meta.Remote.CreatorFileNum,
+			Locator:        meta.Remote.Locator,
+			CleanupMethod:  meta.Remote.CleanupMethod,
+		})
+	} else {
+		p.mu.localObjectsChanged = true
+	}
+}
+
+func (p *provider) removeMetadata(fileNum base.DiskFileNum) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	meta, ok := p.mu.knownObjects[fileNum]
+	if !ok {
+		return
+	}
+	delete(p.mu.knownObjects, fileNum)
+	if meta.IsRemote() {
+		p.mu.remote.catalogBatch.DeleteObject(fileNum)
+	} else {
+		p.mu.localObjectsChanged = true
+	}
+}
+
+// protectObject prevents the unreferencing of a remote object until
+// unprotectObject is called.
+func (p *provider) protectObject(fileNum base.DiskFileNum) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.mu.protectedObjects[fileNum] = p.mu.protectedObjects[fileNum] + 1
+}
+
+func (p *provider) unprotectObject(fileNum base.DiskFileNum) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	v := p.mu.protectedObjects[fileNum]
+	if invariants.Enabled && v == 0 {
+		panic("invalid protection count")
+	}
+	if v > 1 {
+		p.mu.protectedObjects[fileNum] = v - 1
+	} else {
+		delete(p.mu.protectedObjects, fileNum)
+		// TODO(radu): check if the object is still in knownObject; if not, unref it
+		// now.
+	}
+}
+
+func (p *provider) isProtected(fileNum base.DiskFileNum) bool {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.mu.protectedObjects[fileNum] > 0
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go

@@ -0,0 +1,206 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import "github.com/cockroachdb/pebble/internal/invariants"
+
+const (
+	// Constants for dynamic readahead of data blocks. Note that the size values
+	// make sense as some multiple of the default block size; and they should
+	// both be larger than the default block size.
+	minFileReadsForReadahead = 2
+	// TODO(bilal): Have the initial size value be a factor of the block size,
+	// as opposed to a hardcoded value.
+	initialReadaheadSize = 64 << 10 /* 64KB */
+)
+
+// readaheadState contains state variables related to readahead. Updated on
+// file reads.
+type readaheadState struct {
+	// Number of sequential reads.
+	numReads         int64
+	maxReadaheadSize int64
+	// Size issued to the next call to Prefetch. Starts at or above
+	// initialReadaheadSize and grows exponentially until maxReadaheadSize.
+	size int64
+	// prevSize is the size used in the last Prefetch call.
+	prevSize int64
+	// The byte offset up to which the OS has been asked to read ahead / cached.
+	// When reading ahead, reads up to this limit should not incur an IO
+	// operation. Reads after this limit can benefit from a new call to
+	// Prefetch.
+	limit int64
+}
+
+func makeReadaheadState(maxReadaheadSize int64) readaheadState {
+	return readaheadState{
+		size:             initialReadaheadSize,
+		maxReadaheadSize: maxReadaheadSize,
+	}
+}
+
+func (rs *readaheadState) recordCacheHit(offset, blockLength int64) {
+	currentReadEnd := offset + blockLength
+	if rs.numReads >= minFileReadsForReadahead {
+		if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
+			// This is a read that would have resulted in a readahead, had it
+			// not been a cache hit.
+			rs.limit = currentReadEnd
+			return
+		}
+		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+rs.maxReadaheadSize {
+			// We read too far away from rs.limit to benefit from readahead in
+			// any scenario. Reset all variables.
+			rs.numReads = 1
+			rs.limit = currentReadEnd
+			rs.size = initialReadaheadSize
+			rs.prevSize = 0
+			return
+		}
+		// Reads in the range [rs.limit - rs.prevSize, rs.limit] end up
+		// here. This is a read that is potentially benefitting from a past
+		// readahead.
+		return
+	}
+	if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
+		// Blocks are being read sequentially and would benefit from readahead
+		// down the line.
+		rs.numReads++
+		return
+	}
+	// We read too far ahead of the last read, or before it. This indicates
+	// a random read, where readahead is not desirable. Reset all variables.
+	rs.numReads = 1
+	rs.limit = currentReadEnd
+	rs.size = initialReadaheadSize
+	rs.prevSize = 0
+}
+
+// maybeReadahead updates state and determines whether to issue a readahead /
+// prefetch call for a block read at offset for blockLength bytes.
+// Returns a size value (greater than 0) that should be prefetched if readahead
+// would be beneficial.
+func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
+	if invariants.Enabled && rs.maxReadaheadSize == 0 {
+		panic("readaheadState not initialized")
+	}
+	currentReadEnd := offset + blockLength
+	if rs.numReads >= minFileReadsForReadahead {
+		// The minimum threshold of sequential reads to justify reading ahead
+		// has been reached.
+		// There are two intervals: the interval being read:
+		// [offset, currentReadEnd]
+		// as well as the interval where a read would benefit from read ahead:
+		// [rs.limit, rs.limit + rs.size]
+		// We increase the latter interval to
+		// [rs.limit, rs.limit + rs.maxReadaheadSize] to account for cases where
+		// readahead may not be beneficial with a small readahead size, but over
+		// time the readahead size would increase exponentially to make it
+		// beneficial.
+		if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
+			// We are doing a read in the interval ahead of
+			// the last readahead range. In the diagrams below, ++++ is the last
+			// readahead range, ==== is the range represented by
+			// [rs.limit, rs.limit + rs.maxReadaheadSize], and ---- is the range
+			// being read.
+			//
+			//               rs.limit           rs.limit + rs.maxReadaheadSize
+			//         ++++++++++|===========================|
+			//
+			//              |-------------|
+			//            offset       currentReadEnd
+			//
+			// This case is also possible, as are all cases with an overlap
+			// between [rs.limit, rs.limit + rs.maxReadaheadSize] and [offset,
+			// currentReadEnd]:
+			//
+			//               rs.limit           rs.limit + rs.maxReadaheadSize
+			//         ++++++++++|===========================|
+			//
+			//                                            |-------------|
+			//                                         offset       currentReadEnd
+			//
+			//
+			rs.numReads++
+			rs.limit = offset + rs.size
+			rs.prevSize = rs.size
+			// Increase rs.size for the next read.
+			rs.size *= 2
+			if rs.size > rs.maxReadaheadSize {
+				rs.size = rs.maxReadaheadSize
+			}
+			return rs.prevSize
+		}
+		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+rs.maxReadaheadSize {
+			// The above conditional has rs.limit > rs.prevSize to confirm that
+			// rs.limit - rs.prevSize would not underflow.
+			// We read too far away from rs.limit to benefit from readahead in
+			// any scenario. Reset all variables.
+			// The case where we read too far ahead:
+			//
+			// (rs.limit - rs.prevSize)    (rs.limit)   (rs.limit + rs.maxReadaheadSize)
+			//                    |+++++++++++++|=============|
+			//
+			//                                                  |-------------|
+			//                                             offset       currentReadEnd
+			//
+			// Or too far behind:
+			//
+			// (rs.limit - rs.prevSize)    (rs.limit)   (rs.limit + rs.maxReadaheadSize)
+			//                    |+++++++++++++|=============|
+			//
+			//    |-------------|
+			// offset       currentReadEnd
+			//
+			rs.numReads = 1
+			rs.limit = currentReadEnd
+			rs.size = initialReadaheadSize
+			rs.prevSize = 0
+
+			return 0
+		}
+		// Reads in the range [rs.limit - rs.prevSize, rs.limit] end up
+		// here. This is a read that is potentially benefitting from a past
+		// readahead, but there's no reason to issue a readahead call at the
+		// moment.
+		//
+		// (rs.limit - rs.prevSize)            (rs.limit + rs.maxReadaheadSize)
+		//                    |+++++++++++++|===============|
+		//                             (rs.limit)
+		//
+		//                        |-------|
+		//                     offset    currentReadEnd
+		//
+		rs.numReads++
+		return 0
+	}
+	if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
+		// Blocks are being read sequentially and would benefit from readahead
+		// down the line.
+		//
+		//                       (rs.limit)   (rs.limit + rs.maxReadaheadSize)
+		//                         |=============|
+		//
+		//                    |-------|
+		//                offset    currentReadEnd
+		//
+		rs.numReads++
+		return 0
+	}
+	// We read too far ahead of the last read, or before it. This indicates
+	// a random read, where readahead is not desirable. Reset all variables.
+	//
+	// (rs.limit - rs.maxReadaheadSize)  (rs.limit)   (rs.limit + rs.maxReadaheadSize)
+	//                     |+++++++++++++|=============|
+	//
+	//                                                    |-------|
+	//                                                offset    currentReadEnd
+	//
+	rs.numReads = 1
+	rs.limit = currentReadEnd
+	rs.size = initialReadaheadSize
+	rs.prevSize = 0
+	return 0
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote.go

@@ -0,0 +1,376 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"sync"
+	"sync/atomic"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+)
+
+// remoteSubsystem contains the provider fields related to remote storage.
+// All fields remain unset if remote storage is not configured.
+type remoteSubsystem struct {
+	catalog *remoteobjcat.Catalog
+	// catalogSyncMutex is used to correctly serialize two sharedSync operations.
+	// It must be acquired before the provider mutex.
+	catalogSyncMutex sync.Mutex
+
+	cache *sharedcache.Cache
+
+	// shared contains the fields relevant to shared objects, i.e. objects that
+	// are created by Pebble and potentially shared between Pebble instances.
+	shared struct {
+		// initialized guards access to the creatorID field.
+		initialized atomic.Bool
+		creatorID   objstorage.CreatorID
+		initOnce    sync.Once
+
+		// checkRefsOnOpen controls whether we check the ref marker file when opening
+		// an object. Normally this is true when invariants are enabled (but the provider
+		// test tweaks this field).
+		checkRefsOnOpen bool
+	}
+}
+
+// remoteInit initializes the remote object subsystem (if configured) and finds
+// any remote objects.
+func (p *provider) remoteInit() error {
+	if p.st.Remote.StorageFactory == nil {
+		return nil
+	}
+	catalog, contents, err := remoteobjcat.Open(p.st.FS, p.st.FSDirName)
+	if err != nil {
+		return errors.Wrapf(err, "pebble: could not open remote object catalog")
+	}
+	p.remote.catalog = catalog
+	p.remote.shared.checkRefsOnOpen = invariants.Enabled
+
+	// The creator ID may or may not be initialized yet.
+	if contents.CreatorID.IsSet() {
+		p.remote.initShared(contents.CreatorID)
+		p.st.Logger.Infof("remote storage configured; creatorID = %s", contents.CreatorID)
+	} else {
+		p.st.Logger.Infof("remote storage configured; no creatorID yet")
+	}
+
+	if p.st.Remote.CacheSizeBytes > 0 {
+		const defaultBlockSize = 32 * 1024
+		blockSize := p.st.Remote.CacheBlockSize
+		if blockSize == 0 {
+			blockSize = defaultBlockSize
+		}
+
+		const defaultShardingBlockSize = 1024 * 1024
+		shardingBlockSize := p.st.Remote.ShardingBlockSize
+		if shardingBlockSize == 0 {
+			shardingBlockSize = defaultShardingBlockSize
+		}
+
+		numShards := p.st.Remote.CacheShardCount
+		if numShards == 0 {
+			numShards = 2 * runtime.GOMAXPROCS(0)
+		}
+
+		p.remote.cache, err = sharedcache.Open(
+			p.st.FS, p.st.Logger, p.st.FSDirName, blockSize, shardingBlockSize, p.st.Remote.CacheSizeBytes, numShards)
+		if err != nil {
+			return errors.Wrapf(err, "pebble: could not open remote object cache")
+		}
+	}
+
+	for _, meta := range contents.Objects {
+		o := objstorage.ObjectMetadata{
+			DiskFileNum: meta.FileNum,
+			FileType:    meta.FileType,
+		}
+		o.Remote.CreatorID = meta.CreatorID
+		o.Remote.CreatorFileNum = meta.CreatorFileNum
+		o.Remote.CleanupMethod = meta.CleanupMethod
+		o.Remote.Locator = meta.Locator
+		o.Remote.CustomObjectName = meta.CustomObjectName
+		o.Remote.Storage, err = p.ensureStorageLocked(o.Remote.Locator)
+		if err != nil {
+			return errors.Wrapf(err, "creating remote.Storage object for locator '%s'", o.Remote.Locator)
+		}
+		if invariants.Enabled {
+			o.AssertValid()
+		}
+		p.mu.knownObjects[o.DiskFileNum] = o
+	}
+	return nil
+}
+
+// initShared initializes the creator ID, allowing use of shared objects.
+func (ss *remoteSubsystem) initShared(creatorID objstorage.CreatorID) {
+	ss.shared.initOnce.Do(func() {
+		ss.shared.creatorID = creatorID
+		ss.shared.initialized.Store(true)
+	})
+}
+
+func (p *provider) sharedClose() error {
+	if p.st.Remote.StorageFactory == nil {
+		return nil
+	}
+	var err error
+	if p.remote.cache != nil {
+		err = p.remote.cache.Close()
+		p.remote.cache = nil
+	}
+	if p.remote.catalog != nil {
+		err = firstError(err, p.remote.catalog.Close())
+		p.remote.catalog = nil
+	}
+	return err
+}
+
+// SetCreatorID is part of the objstorage.Provider interface.
+func (p *provider) SetCreatorID(creatorID objstorage.CreatorID) error {
+	if p.st.Remote.StorageFactory == nil {
+		return errors.AssertionFailedf("attempt to set CreatorID but remote storage not enabled")
+	}
+	// Note: this call is a cheap no-op if the creator ID was already set. This
+	// call also checks if we are trying to change the ID.
+	if err := p.remote.catalog.SetCreatorID(creatorID); err != nil {
+		return err
+	}
+	if !p.remote.shared.initialized.Load() {
+		p.st.Logger.Infof("remote storage creatorID set to %s", creatorID)
+		p.remote.initShared(creatorID)
+	}
+	return nil
+}
+
+// IsSharedForeign is part of the objstorage.Provider interface.
+func (p *provider) IsSharedForeign(meta objstorage.ObjectMetadata) bool {
+	if !p.remote.shared.initialized.Load() {
+		return false
+	}
+	return meta.IsShared() && (meta.Remote.CreatorID != p.remote.shared.creatorID)
+}
+
+func (p *provider) remoteCheckInitialized() error {
+	if p.st.Remote.StorageFactory == nil {
+		return errors.Errorf("remote object support not configured")
+	}
+	return nil
+}
+
+func (p *provider) sharedCheckInitialized() error {
+	if err := p.remoteCheckInitialized(); err != nil {
+		return err
+	}
+	if !p.remote.shared.initialized.Load() {
+		return errors.Errorf("remote object support not available: remote creator ID not yet set")
+	}
+	return nil
+}
+
+func (p *provider) sharedSync() error {
+	// Serialize parallel sync operations. Note that ApplyBatch is already
+	// serialized internally, but we want to make sure they get called with
+	// batches in the right order.
+	p.remote.catalogSyncMutex.Lock()
+	defer p.remote.catalogSyncMutex.Unlock()
+
+	batch := func() remoteobjcat.Batch {
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		res := p.mu.remote.catalogBatch.Copy()
+		p.mu.remote.catalogBatch.Reset()
+		return res
+	}()
+
+	if batch.IsEmpty() {
+		return nil
+	}
+
+	if err := p.remote.catalog.ApplyBatch(batch); err != nil {
+		// Put back the batch (for the next Sync), appending any operations that
+		// happened in the meantime.
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		batch.Append(p.mu.remote.catalogBatch)
+		p.mu.remote.catalogBatch = batch
+		return err
+	}
+
+	return nil
+}
+
+func (p *provider) remotePath(meta objstorage.ObjectMetadata) string {
+	if meta.Remote.Locator != "" {
+		return fmt.Sprintf("remote-%s://%s", meta.Remote.Locator, remoteObjectName(meta))
+	}
+	return "remote://" + remoteObjectName(meta)
+}
+
+// sharedCreateRef creates a reference marker object.
+func (p *provider) sharedCreateRef(meta objstorage.ObjectMetadata) error {
+	if err := p.sharedCheckInitialized(); err != nil {
+		return err
+	}
+	if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
+		return nil
+	}
+	refName := p.sharedObjectRefName(meta)
+	writer, err := meta.Remote.Storage.CreateObject(refName)
+	if err == nil {
+		// The object is empty, just close the writer.
+		err = writer.Close()
+	}
+	if err != nil {
+		return errors.Wrapf(err, "creating marker object %q", refName)
+	}
+	return nil
+}
+
+func (p *provider) sharedCreate(
+	_ context.Context,
+	fileType base.FileType,
+	fileNum base.DiskFileNum,
+	locator remote.Locator,
+	opts objstorage.CreateOptions,
+) (objstorage.Writable, objstorage.ObjectMetadata, error) {
+	if err := p.sharedCheckInitialized(); err != nil {
+		return nil, objstorage.ObjectMetadata{}, err
+	}
+	storage, err := p.ensureStorage(locator)
+	if err != nil {
+		return nil, objstorage.ObjectMetadata{}, err
+	}
+	meta := objstorage.ObjectMetadata{
+		DiskFileNum: fileNum,
+		FileType:    fileType,
+	}
+	meta.Remote.CreatorID = p.remote.shared.creatorID
+	meta.Remote.CreatorFileNum = fileNum
+	meta.Remote.CleanupMethod = opts.SharedCleanupMethod
+	meta.Remote.Locator = locator
+	meta.Remote.Storage = storage
+
+	objName := remoteObjectName(meta)
+	writer, err := storage.CreateObject(objName)
+	if err != nil {
+		return nil, objstorage.ObjectMetadata{}, errors.Wrapf(err, "creating object %q", objName)
+	}
+	return &sharedWritable{
+		p:             p,
+		meta:          meta,
+		storageWriter: writer,
+	}, meta, nil
+}
+
+func (p *provider) remoteOpenForReading(
+	ctx context.Context, meta objstorage.ObjectMetadata, opts objstorage.OpenOptions,
+) (objstorage.Readable, error) {
+	if err := p.remoteCheckInitialized(); err != nil {
+		return nil, err
+	}
+	// Verify we have a reference on this object; for performance reasons, we only
+	// do this in testing scenarios.
+	if p.remote.shared.checkRefsOnOpen && meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
+		if err := p.sharedCheckInitialized(); err != nil {
+			return nil, err
+		}
+		refName := p.sharedObjectRefName(meta)
+		if _, err := meta.Remote.Storage.Size(refName); err != nil {
+			if meta.Remote.Storage.IsNotExistError(err) {
+				if opts.MustExist {
+					p.st.Logger.Fatalf("marker object %q does not exist", refName)
+					// TODO(radu): maybe list references for the object.
+				}
+				return nil, errors.Errorf("marker object %q does not exist", refName)
+			}
+			return nil, errors.Wrapf(err, "checking marker object %q", refName)
+		}
+	}
+	objName := remoteObjectName(meta)
+	reader, size, err := meta.Remote.Storage.ReadObject(ctx, objName)
+	if err != nil {
+		if opts.MustExist && meta.Remote.Storage.IsNotExistError(err) {
+			p.st.Logger.Fatalf("object %q does not exist", objName)
+			// TODO(radu): maybe list references for the object.
+		}
+		return nil, err
+	}
+	return p.newRemoteReadable(reader, size, meta.DiskFileNum), nil
+}
+
+func (p *provider) remoteSize(meta objstorage.ObjectMetadata) (int64, error) {
+	if err := p.remoteCheckInitialized(); err != nil {
+		return 0, err
+	}
+	objName := remoteObjectName(meta)
+	return meta.Remote.Storage.Size(objName)
+}
+
+// sharedUnref implements object "removal" with the remote backend. The ref
+// marker object is removed and the backing object is removed only if there are
+// no other ref markers.
+func (p *provider) sharedUnref(meta objstorage.ObjectMetadata) error {
+	if meta.Remote.CleanupMethod == objstorage.SharedNoCleanup {
+		// Never delete objects in this mode.
+		return nil
+	}
+	if p.isProtected(meta.DiskFileNum) {
+		// TODO(radu): we need a mechanism to unref the object when it becomes
+		// unprotected.
+		return nil
+	}
+
+	refName := p.sharedObjectRefName(meta)
+	// Tolerate a not-exists error.
+	if err := meta.Remote.Storage.Delete(refName); err != nil && !meta.Remote.Storage.IsNotExistError(err) {
+		return err
+	}
+	otherRefs, err := meta.Remote.Storage.List(sharedObjectRefPrefix(meta), "" /* delimiter */)
+	if err != nil {
+		return err
+	}
+	if len(otherRefs) == 0 {
+		objName := remoteObjectName(meta)
+		if err := meta.Remote.Storage.Delete(objName); err != nil && !meta.Remote.Storage.IsNotExistError(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+// ensureStorageLocked populates the remote.Storage object for the given
+// locator, if necessary. p.mu must be held.
+func (p *provider) ensureStorageLocked(locator remote.Locator) (remote.Storage, error) {
+	if p.mu.remote.storageObjects == nil {
+		p.mu.remote.storageObjects = make(map[remote.Locator]remote.Storage)
+	}
+	if res, ok := p.mu.remote.storageObjects[locator]; ok {
+		return res, nil
+	}
+	res, err := p.st.Remote.StorageFactory.CreateStorage(locator)
+	if err != nil {
+		return nil, err
+	}
+
+	p.mu.remote.storageObjects[locator] = res
+	return res, nil
+}
+
+// ensureStorage populates the remote.Storage object for the given locator, if necessary.
+func (p *provider) ensureStorage(locator remote.Locator) (remote.Storage, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.ensureStorageLocked(locator)
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_backing.go

@@ -0,0 +1,303 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+)
+
+const (
+	tagCreatorID      = 1
+	tagCreatorFileNum = 2
+	tagCleanupMethod  = 3
+	// tagRefCheckID encodes the information for a ref marker that needs to be
+	// checked when attaching this object to another provider. This is set to the
+	// creator ID and FileNum for the provider that encodes the backing, and
+	// allows the "target" provider to check that the "source" provider kept its
+	// reference on the object alive.
+	tagRefCheckID = 4
+	// tagLocator encodes the remote.Locator; if absent the locator is "". It is
+	// followed by the locator string length and the locator string.
+	tagLocator = 5
+	// tagLocator encodes a custom object name (if present). It is followed by the
+	// custom name string length and the string.
+	tagCustomObjectName = 6
+
+	// Any new tags that don't have the tagNotSafeToIgnoreMask bit set must be
+	// followed by the length of the data (so they can be skipped).
+
+	// Any new tags that have the tagNotSafeToIgnoreMask bit set cause errors if
+	// they are encountered by earlier code that doesn't know the tag.
+	tagNotSafeToIgnoreMask = 64
+)
+
+func (p *provider) encodeRemoteObjectBacking(
+	meta *objstorage.ObjectMetadata,
+) (objstorage.RemoteObjectBacking, error) {
+	if !meta.IsRemote() {
+		return nil, errors.AssertionFailedf("object %s not on remote storage", meta.DiskFileNum)
+	}
+
+	buf := make([]byte, 0, binary.MaxVarintLen64*4)
+	buf = binary.AppendUvarint(buf, tagCreatorID)
+	buf = binary.AppendUvarint(buf, uint64(meta.Remote.CreatorID))
+	// TODO(radu): encode file type as well?
+	buf = binary.AppendUvarint(buf, tagCreatorFileNum)
+	buf = binary.AppendUvarint(buf, uint64(meta.Remote.CreatorFileNum.FileNum()))
+	buf = binary.AppendUvarint(buf, tagCleanupMethod)
+	buf = binary.AppendUvarint(buf, uint64(meta.Remote.CleanupMethod))
+	if meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
+		buf = binary.AppendUvarint(buf, tagRefCheckID)
+		buf = binary.AppendUvarint(buf, uint64(p.remote.shared.creatorID))
+		buf = binary.AppendUvarint(buf, uint64(meta.DiskFileNum.FileNum()))
+	}
+	if meta.Remote.Locator != "" {
+		buf = binary.AppendUvarint(buf, tagLocator)
+		buf = encodeString(buf, string(meta.Remote.Locator))
+	}
+	if meta.Remote.CustomObjectName != "" {
+		buf = binary.AppendUvarint(buf, tagCustomObjectName)
+		buf = encodeString(buf, meta.Remote.CustomObjectName)
+	}
+	return buf, nil
+}
+
+type remoteObjectBackingHandle struct {
+	backing objstorage.RemoteObjectBacking
+	fileNum base.DiskFileNum
+	p       *provider
+}
+
+func (s *remoteObjectBackingHandle) Get() (objstorage.RemoteObjectBacking, error) {
+	if s.backing == nil {
+		return nil, errors.Errorf("RemoteObjectBackingHandle.Get() called after Close()")
+	}
+	return s.backing, nil
+}
+
+func (s *remoteObjectBackingHandle) Close() {
+	if s.backing != nil {
+		s.backing = nil
+		s.p.unprotectObject(s.fileNum)
+	}
+}
+
+var _ objstorage.RemoteObjectBackingHandle = (*remoteObjectBackingHandle)(nil)
+
+// RemoteObjectBacking is part of the objstorage.Provider interface.
+func (p *provider) RemoteObjectBacking(
+	meta *objstorage.ObjectMetadata,
+) (objstorage.RemoteObjectBackingHandle, error) {
+	backing, err := p.encodeRemoteObjectBacking(meta)
+	if err != nil {
+		return nil, err
+	}
+	p.protectObject(meta.DiskFileNum)
+	return &remoteObjectBackingHandle{
+		backing: backing,
+		fileNum: meta.DiskFileNum,
+		p:       p,
+	}, nil
+}
+
+// CreateExternalObjectBacking is part of the objstorage.Provider interface.
+func (p *provider) CreateExternalObjectBacking(
+	locator remote.Locator, objName string,
+) (objstorage.RemoteObjectBacking, error) {
+	var meta objstorage.ObjectMetadata
+	meta.Remote.Locator = locator
+	meta.Remote.CustomObjectName = objName
+	meta.Remote.CleanupMethod = objstorage.SharedNoCleanup
+	return p.encodeRemoteObjectBacking(&meta)
+}
+
+type decodedBacking struct {
+	meta objstorage.ObjectMetadata
+	// refToCheck is set only when meta.Remote.CleanupMethod is RefTracking
+	refToCheck struct {
+		creatorID objstorage.CreatorID
+		fileNum   base.DiskFileNum
+	}
+}
+
+// decodeRemoteObjectBacking decodes the remote object metadata.
+//
+// Note that the meta.Remote.Storage field is not set.
+func decodeRemoteObjectBacking(
+	fileType base.FileType, fileNum base.DiskFileNum, buf objstorage.RemoteObjectBacking,
+) (decodedBacking, error) {
+	var creatorID, creatorFileNum, cleanupMethod, refCheckCreatorID, refCheckFileNum uint64
+	var locator, customObjName string
+	br := bytes.NewReader(buf)
+	for {
+		tag, err := binary.ReadUvarint(br)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return decodedBacking{}, err
+		}
+		switch tag {
+		case tagCreatorID:
+			creatorID, err = binary.ReadUvarint(br)
+
+		case tagCreatorFileNum:
+			creatorFileNum, err = binary.ReadUvarint(br)
+
+		case tagCleanupMethod:
+			cleanupMethod, err = binary.ReadUvarint(br)
+
+		case tagRefCheckID:
+			refCheckCreatorID, err = binary.ReadUvarint(br)
+			if err == nil {
+				refCheckFileNum, err = binary.ReadUvarint(br)
+			}
+
+		case tagLocator:
+			locator, err = decodeString(br)
+
+		case tagCustomObjectName:
+			customObjName, err = decodeString(br)
+
+		default:
+			// Ignore unknown tags, unless they're not safe to ignore.
+			if tag&tagNotSafeToIgnoreMask != 0 {
+				return decodedBacking{}, errors.Newf("unknown tag %d", tag)
+			}
+			var dataLen uint64
+			dataLen, err = binary.ReadUvarint(br)
+			if err == nil {
+				_, err = br.Seek(int64(dataLen), io.SeekCurrent)
+			}
+		}
+		if err != nil {
+			return decodedBacking{}, err
+		}
+	}
+	if customObjName == "" {
+		if creatorID == 0 {
+			return decodedBacking{}, errors.Newf("remote object backing missing creator ID")
+		}
+		if creatorFileNum == 0 {
+			return decodedBacking{}, errors.Newf("remote object backing missing creator file num")
+		}
+	}
+	var res decodedBacking
+	res.meta.DiskFileNum = fileNum
+	res.meta.FileType = fileType
+	res.meta.Remote.CreatorID = objstorage.CreatorID(creatorID)
+	res.meta.Remote.CreatorFileNum = base.FileNum(creatorFileNum).DiskFileNum()
+	res.meta.Remote.CleanupMethod = objstorage.SharedCleanupMethod(cleanupMethod)
+	if res.meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
+		if refCheckCreatorID == 0 || refCheckFileNum == 0 {
+			return decodedBacking{}, errors.Newf("remote object backing missing ref to check")
+		}
+		res.refToCheck.creatorID = objstorage.CreatorID(refCheckCreatorID)
+		res.refToCheck.fileNum = base.FileNum(refCheckFileNum).DiskFileNum()
+	}
+	res.meta.Remote.Locator = remote.Locator(locator)
+	res.meta.Remote.CustomObjectName = customObjName
+	return res, nil
+}
+
+func encodeString(buf []byte, s string) []byte {
+	buf = binary.AppendUvarint(buf, uint64(len(s)))
+	buf = append(buf, []byte(s)...)
+	return buf
+}
+
+func decodeString(br io.ByteReader) (string, error) {
+	length, err := binary.ReadUvarint(br)
+	if err != nil || length == 0 {
+		return "", err
+	}
+	buf := make([]byte, length)
+	for i := range buf {
+		buf[i], err = br.ReadByte()
+		if err != nil {
+			return "", err
+		}
+	}
+	return string(buf), nil
+}
+
+// AttachRemoteObjects is part of the objstorage.Provider interface.
+func (p *provider) AttachRemoteObjects(
+	objs []objstorage.RemoteObjectToAttach,
+) ([]objstorage.ObjectMetadata, error) {
+	decoded := make([]decodedBacking, len(objs))
+	for i, o := range objs {
+		var err error
+		decoded[i], err = decodeRemoteObjectBacking(o.FileType, o.FileNum, o.Backing)
+		if err != nil {
+			return nil, err
+		}
+		decoded[i].meta.Remote.Storage, err = p.ensureStorage(decoded[i].meta.Remote.Locator)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Create the reference marker objects.
+	// TODO(radu): parallelize this.
+	for _, d := range decoded {
+		if d.meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
+			continue
+		}
+		if err := p.sharedCreateRef(d.meta); err != nil {
+			// TODO(radu): clean up references previously created in this loop.
+			return nil, err
+		}
+		// Check the "origin"'s reference.
+		refName := sharedObjectRefName(d.meta, d.refToCheck.creatorID, d.refToCheck.fileNum)
+		if _, err := d.meta.Remote.Storage.Size(refName); err != nil {
+			_ = p.sharedUnref(d.meta)
+			// TODO(radu): clean up references previously created in this loop.
+			if d.meta.Remote.Storage.IsNotExistError(err) {
+				return nil, errors.Errorf("origin marker object %q does not exist;"+
+					" object probably removed from the provider which created the backing", refName)
+			}
+			return nil, errors.Wrapf(err, "checking origin's marker object %s", refName)
+		}
+	}
+
+	func() {
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		for _, d := range decoded {
+			p.mu.remote.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
+				FileNum:        d.meta.DiskFileNum,
+				FileType:       d.meta.FileType,
+				CreatorID:      d.meta.Remote.CreatorID,
+				CreatorFileNum: d.meta.Remote.CreatorFileNum,
+				CleanupMethod:  d.meta.Remote.CleanupMethod,
+				Locator:        d.meta.Remote.Locator,
+			})
+		}
+	}()
+	if err := p.sharedSync(); err != nil {
+		return nil, err
+	}
+
+	metas := make([]objstorage.ObjectMetadata, len(decoded))
+	for i, d := range decoded {
+		metas[i] = d.meta
+	}
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	for _, meta := range metas {
+		p.mu.knownObjects[meta.DiskFileNum] = meta
+	}
+	return metas, nil
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_obj_name.go

@@ -0,0 +1,91 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"fmt"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+)
+
+// remoteObjectName returns the name of an object on remote storage.
+//
+// For sstables, the format is: <hash>-<creator-id>-<file-num>.sst
+// For example: 1a3f-2-000001.sst
+func remoteObjectName(meta objstorage.ObjectMetadata) string {
+	if meta.Remote.CustomObjectName != "" {
+		return meta.Remote.CustomObjectName
+	}
+	switch meta.FileType {
+	case base.FileTypeTable:
+		return fmt.Sprintf(
+			"%04x-%d-%06d.sst",
+			objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(),
+		)
+	}
+	panic("unknown FileType")
+}
+
+// sharedObjectRefName returns the name of the object's ref marker associated
+// with a given referencing provider. This name is the object's name concatenated with
+// ".ref.<ref-creator-id>.<local-file-num>".
+//
+// For example: 1a3f-2-000001.sst.ref.5.000008
+func sharedObjectRefName(
+	meta objstorage.ObjectMetadata, refCreatorID objstorage.CreatorID, refFileNum base.DiskFileNum,
+) string {
+	if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
+		panic("ref object used when ref tracking disabled")
+	}
+	if meta.Remote.CustomObjectName != "" {
+		return fmt.Sprintf(
+			"%s.ref.%d.%06d", meta.Remote.CustomObjectName, refCreatorID, refFileNum.FileNum(),
+		)
+	}
+	switch meta.FileType {
+	case base.FileTypeTable:
+		return fmt.Sprintf(
+			"%04x-%d-%06d.sst.ref.%d.%06d",
+			objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(), refCreatorID, refFileNum.FileNum(),
+		)
+	}
+	panic("unknown FileType")
+}
+
+func sharedObjectRefPrefix(meta objstorage.ObjectMetadata) string {
+	if meta.Remote.CustomObjectName != "" {
+		return meta.Remote.CustomObjectName + ".ref."
+	}
+	switch meta.FileType {
+	case base.FileTypeTable:
+		return fmt.Sprintf(
+			"%04x-%d-%06d.sst.ref.",
+			objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(),
+		)
+	}
+	panic("unknown FileType")
+}
+
+// sharedObjectRefName returns the name of the object's ref marker associated
+// with this provider. This name is the object's name concatenated with
+// ".ref.<creator-id>.<local-file-num>".
+//
+// For example: 1a3f-2-000001.sst.ref.5.000008
+func (p *provider) sharedObjectRefName(meta objstorage.ObjectMetadata) string {
+	if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
+		panic("ref object used when ref tracking disabled")
+	}
+	return sharedObjectRefName(meta, p.remote.shared.creatorID, meta.DiskFileNum)
+}
+
+// objHash returns a 16-bit hash value derived from the creator ID and creator
+// file num. We prepend this value to object names to ensure balanced
+// partitioning with AWS (and likely other blob storage providers).
+func objHash(meta objstorage.ObjectMetadata) uint16 {
+	const prime1 = 7459
+	const prime2 = 17539
+	return uint16(uint64(meta.Remote.CreatorID)*prime1 + uint64(meta.Remote.CreatorFileNum.FileNum())*prime2)
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_readable.go

@@ -0,0 +1,162 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"io"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+)
+
+const remoteMaxReadaheadSize = 1024 * 1024 /* 1MB */
+
+// remoteReadable is a very simple implementation of Readable on top of the
+// ReadCloser returned by remote.Storage.CreateObject.
+type remoteReadable struct {
+	objReader remote.ObjectReader
+	size      int64
+	fileNum   base.DiskFileNum
+	provider  *provider
+}
+
+var _ objstorage.Readable = (*remoteReadable)(nil)
+
+func (p *provider) newRemoteReadable(
+	objReader remote.ObjectReader, size int64, fileNum base.DiskFileNum,
+) *remoteReadable {
+	return &remoteReadable{
+		objReader: objReader,
+		size:      size,
+		fileNum:   fileNum,
+		provider:  p,
+	}
+}
+
+// ReadAt is part of the objstorage.Readable interface.
+func (r *remoteReadable) ReadAt(ctx context.Context, p []byte, offset int64) error {
+	return r.readInternal(ctx, p, offset, false /* forCompaction */)
+}
+
+// readInternal performs a read for the object, using the cache when
+// appropriate.
+func (r *remoteReadable) readInternal(
+	ctx context.Context, p []byte, offset int64, forCompaction bool,
+) error {
+	if cache := r.provider.remote.cache; cache != nil {
+		flags := sharedcache.ReadFlags{
+			// Don't add data to the cache if this read is for a compaction.
+			ReadOnly: forCompaction,
+		}
+		return r.provider.remote.cache.ReadAt(ctx, r.fileNum, p, offset, r.objReader, r.size, flags)
+	}
+	return r.objReader.ReadAt(ctx, p, offset)
+}
+
+func (r *remoteReadable) Close() error {
+	defer func() { r.objReader = nil }()
+	return r.objReader.Close()
+}
+
+func (r *remoteReadable) Size() int64 {
+	return r.size
+}
+
+func (r *remoteReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
+	// TODO(radu): use a pool.
+	rh := &remoteReadHandle{readable: r}
+	rh.readahead.state = makeReadaheadState(remoteMaxReadaheadSize)
+	return rh
+}
+
+type remoteReadHandle struct {
+	readable  *remoteReadable
+	readahead struct {
+		state  readaheadState
+		data   []byte
+		offset int64
+	}
+	forCompaction bool
+}
+
+var _ objstorage.ReadHandle = (*remoteReadHandle)(nil)
+
+// ReadAt is part of the objstorage.ReadHandle interface.
+func (r *remoteReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) error {
+	readaheadSize := r.maybeReadahead(offset, len(p))
+
+	// Check if we already have the data from a previous read-ahead.
+	if rhSize := int64(len(r.readahead.data)); rhSize > 0 {
+		if r.readahead.offset <= offset && r.readahead.offset+rhSize > offset {
+			n := copy(p, r.readahead.data[offset-r.readahead.offset:])
+			if n == len(p) {
+				// All data was available.
+				return nil
+			}
+			// Use the data that we had and do a shorter read.
+			offset += int64(n)
+			p = p[n:]
+			readaheadSize -= n
+		}
+	}
+
+	if readaheadSize > len(p) {
+		// Don't try to read past EOF.
+		if offset+int64(readaheadSize) > r.readable.size {
+			readaheadSize = int(r.readable.size - offset)
+			if readaheadSize <= 0 {
+				// This shouldn't happen in practice (Pebble should never try to read
+				// past EOF).
+				return io.EOF
+			}
+		}
+		r.readahead.offset = offset
+		// TODO(radu): we need to somehow account for this memory.
+		if cap(r.readahead.data) >= readaheadSize {
+			r.readahead.data = r.readahead.data[:readaheadSize]
+		} else {
+			r.readahead.data = make([]byte, readaheadSize)
+		}
+
+		if err := r.readable.readInternal(ctx, r.readahead.data, offset, r.forCompaction); err != nil {
+			// Make sure we don't treat the data as valid next time.
+			r.readahead.data = r.readahead.data[:0]
+			return err
+		}
+		copy(p, r.readahead.data)
+		return nil
+	}
+
+	return r.readable.readInternal(ctx, p, offset, r.forCompaction)
+}
+
+func (r *remoteReadHandle) maybeReadahead(offset int64, len int) int {
+	if r.forCompaction {
+		return remoteMaxReadaheadSize
+	}
+	return int(r.readahead.state.maybeReadahead(offset, int64(len)))
+}
+
+// Close is part of the objstorage.ReadHandle interface.
+func (r *remoteReadHandle) Close() error {
+	r.readable = nil
+	r.readahead.data = nil
+	return nil
+}
+
+// SetupForCompaction is part of the objstorage.ReadHandle interface.
+func (r *remoteReadHandle) SetupForCompaction() {
+	r.forCompaction = true
+}
+
+// RecordCacheHit is part of the objstorage.ReadHandle interface.
+func (r *remoteReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
+	if !r.forCompaction {
+		r.readahead.state.recordCacheHit(offset, size)
+	}
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/catalog.go

@@ -0,0 +1,387 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package remoteobjcat
+
+import (
+	"fmt"
+	"io"
+	"sort"
+	"sync"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+	"github.com/cockroachdb/pebble/record"
+	"github.com/cockroachdb/pebble/vfs"
+	"github.com/cockroachdb/pebble/vfs/atomicfs"
+)
+
+// Catalog is used to manage the on-disk remote object catalog.
+//
+// The catalog file is a log of records, where each record is an encoded
+// VersionEdit.
+type Catalog struct {
+	fs      vfs.FS
+	dirname string
+	mu      struct {
+		sync.Mutex
+
+		creatorID objstorage.CreatorID
+		objects   map[base.DiskFileNum]RemoteObjectMetadata
+
+		marker *atomicfs.Marker
+
+		catalogFile      vfs.File
+		catalogRecWriter *record.Writer
+
+		rotationHelper record.RotationHelper
+
+		// catalogFilename is the filename of catalogFile when catalogFile != nil, otherwise
+		// it is the filename of the last catalog file.
+		catalogFilename string
+	}
+}
+
+// RemoteObjectMetadata encapsulates the data stored in the catalog file for each object.
+type RemoteObjectMetadata struct {
+	// FileNum is the identifier for the object within the context of a single DB
+	// instance.
+	FileNum base.DiskFileNum
+	// FileType is the type of the object. Only certain FileTypes are possible.
+	FileType base.FileType
+	// CreatorID identifies the DB instance that originally created the object.
+	CreatorID objstorage.CreatorID
+	// CreatorFileNum is the identifier for the object within the context of the
+	// DB instance that originally created the object.
+	CreatorFileNum base.DiskFileNum
+	// CleanupMethod indicates the method for cleaning up unused shared objects.
+	CleanupMethod objstorage.SharedCleanupMethod
+	// Locator identifies a remote.Storage implementation.
+	Locator remote.Locator
+	// CustomObjectName (if it is set) overrides the object name that is normally
+	// derived from the CreatorID and CreatorFileNum.
+	CustomObjectName string
+}
+
+const (
+	catalogFilenameBase = "REMOTE-OBJ-CATALOG"
+	catalogMarkerName   = "remote-obj-catalog"
+
+	// We create a new file when the size exceeds 1MB (and some other conditions
+	// hold; see record.RotationHelper).
+	rotateFileSize = 1024 * 1024 // 1MB
+)
+
+// CatalogContents contains the remote objects in the catalog.
+type CatalogContents struct {
+	// CreatorID, if it is set.
+	CreatorID objstorage.CreatorID
+	Objects   []RemoteObjectMetadata
+}
+
+// Open creates a Catalog and loads any existing catalog file, returning the
+// creator ID (if it is set) and the contents.
+func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
+	c := &Catalog{
+		fs:      fs,
+		dirname: dirname,
+	}
+	c.mu.objects = make(map[base.DiskFileNum]RemoteObjectMetadata)
+
+	var err error
+	c.mu.marker, c.mu.catalogFilename, err = atomicfs.LocateMarker(fs, dirname, catalogMarkerName)
+	if err != nil {
+		return nil, CatalogContents{}, err
+	}
+	// If the filename is empty, there is no existing catalog.
+	if c.mu.catalogFilename != "" {
+		if err := c.loadFromCatalogFile(c.mu.catalogFilename); err != nil {
+			return nil, CatalogContents{}, err
+		}
+		if err := c.mu.marker.RemoveObsolete(); err != nil {
+			return nil, CatalogContents{}, err
+		}
+		// TODO(radu): remove obsolete catalog files.
+	}
+	res := CatalogContents{
+		CreatorID: c.mu.creatorID,
+		Objects:   make([]RemoteObjectMetadata, 0, len(c.mu.objects)),
+	}
+	for _, meta := range c.mu.objects {
+		res.Objects = append(res.Objects, meta)
+	}
+	// Sort the objects so the function is deterministic.
+	sort.Slice(res.Objects, func(i, j int) bool {
+		return res.Objects[i].FileNum.FileNum() < res.Objects[j].FileNum.FileNum()
+	})
+	return c, res, nil
+}
+
+// SetCreatorID sets the creator ID. If it is already set, it must match.
+func (c *Catalog) SetCreatorID(id objstorage.CreatorID) error {
+	if !id.IsSet() {
+		return errors.AssertionFailedf("attempt to unset CreatorID")
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.mu.creatorID.IsSet() {
+		if c.mu.creatorID != id {
+			return errors.AssertionFailedf("attempt to change CreatorID from %s to %s", c.mu.creatorID, id)
+		}
+		return nil
+	}
+
+	ve := VersionEdit{CreatorID: id}
+	if err := c.writeToCatalogFileLocked(&ve); err != nil {
+		return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
+	}
+	c.mu.creatorID = id
+	return nil
+}
+
+// Close any open files.
+func (c *Catalog) Close() error {
+	return c.closeCatalogFile()
+}
+
+func (c *Catalog) closeCatalogFile() error {
+	if c.mu.catalogFile == nil {
+		return nil
+	}
+	err1 := c.mu.catalogRecWriter.Close()
+	err2 := c.mu.catalogFile.Close()
+	c.mu.catalogRecWriter = nil
+	c.mu.catalogFile = nil
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+// Batch is used to perform multiple object additions/deletions at once.
+type Batch struct {
+	ve VersionEdit
+}
+
+// AddObject adds a new object to the batch.
+//
+// The given FileNum must be new - it must not match that of any object that was
+// ever in the catalog.
+func (b *Batch) AddObject(meta RemoteObjectMetadata) {
+	b.ve.NewObjects = append(b.ve.NewObjects, meta)
+}
+
+// DeleteObject adds an object removal to the batch.
+func (b *Batch) DeleteObject(fileNum base.DiskFileNum) {
+	b.ve.DeletedObjects = append(b.ve.DeletedObjects, fileNum)
+}
+
+// Reset clears the batch.
+func (b *Batch) Reset() {
+	b.ve.NewObjects = b.ve.NewObjects[:0]
+	b.ve.DeletedObjects = b.ve.DeletedObjects[:0]
+}
+
+// IsEmpty returns true if the batch is empty.
+func (b *Batch) IsEmpty() bool {
+	return len(b.ve.NewObjects) == 0 && len(b.ve.DeletedObjects) == 0
+}
+
+// Copy returns a copy of the Batch.
+func (b *Batch) Copy() Batch {
+	var res Batch
+	if len(b.ve.NewObjects) > 0 {
+		res.ve.NewObjects = make([]RemoteObjectMetadata, len(b.ve.NewObjects))
+		copy(res.ve.NewObjects, b.ve.NewObjects)
+	}
+	if len(b.ve.DeletedObjects) > 0 {
+		res.ve.DeletedObjects = make([]base.DiskFileNum, len(b.ve.DeletedObjects))
+		copy(res.ve.DeletedObjects, b.ve.DeletedObjects)
+	}
+	return res
+}
+
+// Append merges two batches.
+func (b *Batch) Append(other Batch) {
+	b.ve.NewObjects = append(b.ve.NewObjects, other.ve.NewObjects...)
+	b.ve.DeletedObjects = append(b.ve.DeletedObjects, other.ve.DeletedObjects...)
+}
+
+// ApplyBatch applies a batch of updates; returns after the change is stably
+// recorded on storage.
+func (c *Catalog) ApplyBatch(b Batch) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// Sanity checks.
+	toAdd := make(map[base.DiskFileNum]struct{}, len(b.ve.NewObjects))
+	exists := func(n base.DiskFileNum) bool {
+		_, ok := c.mu.objects[n]
+		if !ok {
+			_, ok = toAdd[n]
+		}
+		return ok
+	}
+	for _, meta := range b.ve.NewObjects {
+		if exists(meta.FileNum) {
+			return errors.AssertionFailedf("adding existing object %s", meta.FileNum)
+		}
+		toAdd[meta.FileNum] = struct{}{}
+	}
+	for _, n := range b.ve.DeletedObjects {
+		if !exists(n) {
+			return errors.AssertionFailedf("deleting non-existent object %s", n)
+		}
+	}
+
+	if err := c.writeToCatalogFileLocked(&b.ve); err != nil {
+		return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
+	}
+
+	// Add new objects before deleting any objects. This allows for cases where
+	// the same batch adds and deletes an object.
+	for _, meta := range b.ve.NewObjects {
+		c.mu.objects[meta.FileNum] = meta
+	}
+	for _, n := range b.ve.DeletedObjects {
+		delete(c.mu.objects, n)
+	}
+
+	return nil
+}
+
+func (c *Catalog) loadFromCatalogFile(filename string) error {
+	catalogPath := c.fs.PathJoin(c.dirname, filename)
+	f, err := c.fs.Open(catalogPath)
+	if err != nil {
+		return errors.Wrapf(
+			err, "pebble: could not open remote object catalog file %q for DB %q",
+			errors.Safe(filename), c.dirname,
+		)
+	}
+	defer f.Close()
+	rr := record.NewReader(f, 0 /* logNum */)
+	for {
+		r, err := rr.Next()
+		if err == io.EOF || record.IsInvalidRecord(err) {
+			break
+		}
+		if err != nil {
+			return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
+				errors.Safe(filename))
+		}
+		var ve VersionEdit
+		if err := ve.Decode(r); err != nil {
+			return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
+				errors.Safe(filename))
+		}
+		// Apply the version edit to the current state.
+		if err := ve.Apply(&c.mu.creatorID, c.mu.objects); err != nil {
+			return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
+				errors.Safe(filename))
+		}
+	}
+	return nil
+}
+
+// writeToCatalogFileLocked writes a VersionEdit to the catalog file.
+// Creates a new file if this is the first write.
+func (c *Catalog) writeToCatalogFileLocked(ve *VersionEdit) error {
+	c.mu.rotationHelper.AddRecord(int64(len(ve.NewObjects) + len(ve.DeletedObjects)))
+	snapshotSize := int64(len(c.mu.objects))
+
+	var shouldRotate bool
+	if c.mu.catalogFile == nil {
+		shouldRotate = true
+	} else if c.mu.catalogRecWriter.Size() >= rotateFileSize {
+		shouldRotate = c.mu.rotationHelper.ShouldRotate(snapshotSize)
+	}
+
+	if shouldRotate {
+		if c.mu.catalogFile != nil {
+			if err := c.closeCatalogFile(); err != nil {
+				return err
+			}
+		}
+		if err := c.createNewCatalogFileLocked(); err != nil {
+			return err
+		}
+		c.mu.rotationHelper.Rotate(snapshotSize)
+	}
+	return writeRecord(ve, c.mu.catalogFile, c.mu.catalogRecWriter)
+}
+
+func makeCatalogFilename(iter uint64) string {
+	return fmt.Sprintf("%s-%06d", catalogFilenameBase, iter)
+}
+
+// createNewCatalogFileLocked creates a new catalog file, populates it with the
+// current catalog and sets c.mu.catalogFile and c.mu.catalogRecWriter.
+func (c *Catalog) createNewCatalogFileLocked() (outErr error) {
+	if c.mu.catalogFile != nil {
+		return errors.AssertionFailedf("catalogFile already open")
+	}
+	filename := makeCatalogFilename(c.mu.marker.NextIter())
+	filepath := c.fs.PathJoin(c.dirname, filename)
+	file, err := c.fs.Create(filepath)
+	if err != nil {
+		return err
+	}
+	recWriter := record.NewWriter(file)
+	err = func() error {
+		// Create a VersionEdit that gets us from an empty catalog to the current state.
+		var ve VersionEdit
+		ve.CreatorID = c.mu.creatorID
+		ve.NewObjects = make([]RemoteObjectMetadata, 0, len(c.mu.objects))
+		for _, meta := range c.mu.objects {
+			ve.NewObjects = append(ve.NewObjects, meta)
+		}
+		if err := writeRecord(&ve, file, recWriter); err != nil {
+			return err
+		}
+
+		// Move the marker to the new filename. Move handles syncing the data
+		// directory as well.
+		if err := c.mu.marker.Move(filename); err != nil {
+			return errors.Wrap(err, "moving marker")
+		}
+
+		return nil
+	}()
+
+	if err != nil {
+		_ = recWriter.Close()
+		_ = file.Close()
+		_ = c.fs.Remove(filepath)
+		return err
+	}
+
+	// Remove any previous file (ignoring any error).
+	if c.mu.catalogFilename != "" {
+		_ = c.fs.Remove(c.fs.PathJoin(c.dirname, c.mu.catalogFilename))
+	}
+
+	c.mu.catalogFile = file
+	c.mu.catalogRecWriter = recWriter
+	c.mu.catalogFilename = filename
+	return nil
+}
+
+func writeRecord(ve *VersionEdit, file vfs.File, recWriter *record.Writer) error {
+	w, err := recWriter.Next()
+	if err != nil {
+		return err
+	}
+	if err := ve.Encode(w); err != nil {
+		return err
+	}
+	if err := recWriter.Flush(); err != nil {
+		return err
+	}
+	return file.Sync()
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/version_edit.go

@@ -0,0 +1,254 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package remoteobjcat
+
+import (
+	"bufio"
+	"encoding/binary"
+	"io"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+)
+
+// VersionEdit is a modification to the remote object state which can be encoded
+// into a record.
+//
+// TODO(radu): consider adding creation and deletion time for debugging purposes.
+type VersionEdit struct {
+	NewObjects     []RemoteObjectMetadata
+	DeletedObjects []base.DiskFileNum
+	CreatorID      objstorage.CreatorID
+}
+
+const (
+	// tagNewObject is followed by the FileNum, creator ID, creator FileNum,
+	// cleanup method, optional new object tags, and ending with a 0 byte.
+	tagNewObject = 1
+	// tagDeletedObject is followed by the FileNum.
+	tagDeletedObject = 2
+	// tagCreatorID is followed by the Creator ID for this store. This ID can
+	// never change.
+	tagCreatorID = 3
+	// tagNewObjectLocator is an optional tag inside the tagNewObject payload. It
+	// is followed by the encoded length of the locator string and the string.
+	tagNewObjectLocator = 4
+	// tagNewObjectCustomName is an optional tag inside the tagNewObject payload.
+	// It is followed by the encoded length of the custom object name string
+	// followed by the string.
+	tagNewObjectCustomName = 5
+)
+
+// Object type values. We don't want to encode FileType directly because it is
+// more general (and we want freedom to change it in the future).
+const (
+	objTypeTable = 1
+)
+
+func objTypeToFileType(objType uint64) (base.FileType, error) {
+	switch objType {
+	case objTypeTable:
+		return base.FileTypeTable, nil
+	default:
+		return 0, errors.Newf("unknown object type %d", objType)
+	}
+}
+
+func fileTypeToObjType(fileType base.FileType) (uint64, error) {
+	switch fileType {
+	case base.FileTypeTable:
+		return objTypeTable, nil
+
+	default:
+		return 0, errors.Newf("unknown object type for file type %d", fileType)
+	}
+}
+
+// Encode encodes an edit to the specified writer.
+func (v *VersionEdit) Encode(w io.Writer) error {
+	buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*10+len(v.DeletedObjects)*2+2))
+	for _, meta := range v.NewObjects {
+		objType, err := fileTypeToObjType(meta.FileType)
+		if err != nil {
+			return err
+		}
+		buf = binary.AppendUvarint(buf, uint64(tagNewObject))
+		buf = binary.AppendUvarint(buf, uint64(meta.FileNum.FileNum()))
+		buf = binary.AppendUvarint(buf, objType)
+		buf = binary.AppendUvarint(buf, uint64(meta.CreatorID))
+		buf = binary.AppendUvarint(buf, uint64(meta.CreatorFileNum.FileNum()))
+		buf = binary.AppendUvarint(buf, uint64(meta.CleanupMethod))
+		if meta.Locator != "" {
+			buf = binary.AppendUvarint(buf, uint64(tagNewObjectLocator))
+			buf = encodeString(buf, string(meta.Locator))
+		}
+		if meta.CustomObjectName != "" {
+			buf = binary.AppendUvarint(buf, uint64(tagNewObjectCustomName))
+			buf = encodeString(buf, meta.CustomObjectName)
+		}
+		// Append 0 as the terminator for optional new object tags.
+		buf = binary.AppendUvarint(buf, 0)
+	}
+
+	for _, dfn := range v.DeletedObjects {
+		buf = binary.AppendUvarint(buf, uint64(tagDeletedObject))
+		buf = binary.AppendUvarint(buf, uint64(dfn.FileNum()))
+	}
+	if v.CreatorID.IsSet() {
+		buf = binary.AppendUvarint(buf, uint64(tagCreatorID))
+		buf = binary.AppendUvarint(buf, uint64(v.CreatorID))
+	}
+	_, err := w.Write(buf)
+	return err
+}
+
+// Decode decodes an edit from the specified reader.
+func (v *VersionEdit) Decode(r io.Reader) error {
+	br, ok := r.(io.ByteReader)
+	if !ok {
+		br = bufio.NewReader(r)
+	}
+	for {
+		tag, err := binary.ReadUvarint(br)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		err = nil
+		switch tag {
+		case tagNewObject:
+			var fileNum, creatorID, creatorFileNum, cleanupMethod uint64
+			var locator, customName string
+			var fileType base.FileType
+			fileNum, err = binary.ReadUvarint(br)
+			if err == nil {
+				var objType uint64
+				objType, err = binary.ReadUvarint(br)
+				if err == nil {
+					fileType, err = objTypeToFileType(objType)
+				}
+			}
+			if err == nil {
+				creatorID, err = binary.ReadUvarint(br)
+			}
+			if err == nil {
+				creatorFileNum, err = binary.ReadUvarint(br)
+			}
+			if err == nil {
+				cleanupMethod, err = binary.ReadUvarint(br)
+			}
+			for err == nil {
+				var optionalTag uint64
+				optionalTag, err = binary.ReadUvarint(br)
+				if err != nil || optionalTag == 0 {
+					break
+				}
+
+				switch optionalTag {
+				case tagNewObjectLocator:
+					locator, err = decodeString(br)
+
+				case tagNewObjectCustomName:
+					customName, err = decodeString(br)
+
+				default:
+					err = errors.Newf("unknown newObject tag %d", optionalTag)
+				}
+			}
+
+			if err == nil {
+				v.NewObjects = append(v.NewObjects, RemoteObjectMetadata{
+					FileNum:          base.FileNum(fileNum).DiskFileNum(),
+					FileType:         fileType,
+					CreatorID:        objstorage.CreatorID(creatorID),
+					CreatorFileNum:   base.FileNum(creatorFileNum).DiskFileNum(),
+					CleanupMethod:    objstorage.SharedCleanupMethod(cleanupMethod),
+					Locator:          remote.Locator(locator),
+					CustomObjectName: customName,
+				})
+			}
+
+		case tagDeletedObject:
+			var fileNum uint64
+			fileNum, err = binary.ReadUvarint(br)
+			if err == nil {
+				v.DeletedObjects = append(v.DeletedObjects, base.FileNum(fileNum).DiskFileNum())
+			}
+
+		case tagCreatorID:
+			var id uint64
+			id, err = binary.ReadUvarint(br)
+			if err == nil {
+				v.CreatorID = objstorage.CreatorID(id)
+			}
+
+		default:
+			err = errors.Newf("unknown tag %d", tag)
+		}
+
+		if err != nil {
+			if err == io.EOF {
+				return errCorruptCatalog
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+func encodeString(buf []byte, s string) []byte {
+	buf = binary.AppendUvarint(buf, uint64(len(s)))
+	buf = append(buf, []byte(s)...)
+	return buf
+}
+
+func decodeString(br io.ByteReader) (string, error) {
+	length, err := binary.ReadUvarint(br)
+	if err != nil || length == 0 {
+		return "", err
+	}
+	buf := make([]byte, length)
+	for i := range buf {
+		buf[i], err = br.ReadByte()
+		if err != nil {
+			return "", err
+		}
+	}
+	return string(buf), nil
+}
+
+var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt remote object catalog")
+
+// Apply the version edit to a creator ID and a map of objects.
+func (v *VersionEdit) Apply(
+	creatorID *objstorage.CreatorID, objects map[base.DiskFileNum]RemoteObjectMetadata,
+) error {
+	if v.CreatorID.IsSet() {
+		*creatorID = v.CreatorID
+	}
+	for _, meta := range v.NewObjects {
+		if invariants.Enabled {
+			if _, exists := objects[meta.FileNum]; exists {
+				return errors.AssertionFailedf("version edit adds existing object %s", meta.FileNum)
+			}
+		}
+		objects[meta.FileNum] = meta
+	}
+	for _, fileNum := range v.DeletedObjects {
+		if invariants.Enabled {
+			if _, exists := objects[fileNum]; !exists {
+				return errors.AssertionFailedf("version edit deletes non-existent object %s", fileNum)
+			}
+		}
+		delete(objects, fileNum)
+	}
+	return nil
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_writable.go

@@ -0,0 +1,65 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"io"
+
+	"github.com/cockroachdb/pebble/objstorage"
+)
+
+// NewRemoteWritable creates an objstorage.Writable out of an io.WriteCloser.
+func NewRemoteWritable(obj io.WriteCloser) objstorage.Writable {
+	return &sharedWritable{storageWriter: obj}
+}
+
+// sharedWritable is a very simple implementation of Writable on top of the
+// WriteCloser returned by remote.Storage.CreateObject.
+type sharedWritable struct {
+	// Either both p and meta must be unset / zero values, or both must be set.
+	// The case where both are unset is true only in tests.
+	p             *provider
+	meta          objstorage.ObjectMetadata
+	storageWriter io.WriteCloser
+}
+
+var _ objstorage.Writable = (*sharedWritable)(nil)
+
+// Write is part of the Writable interface.
+func (w *sharedWritable) Write(p []byte) error {
+	_, err := w.storageWriter.Write(p)
+	return err
+}
+
+// Finish is part of the Writable interface.
+func (w *sharedWritable) Finish() error {
+	err := w.storageWriter.Close()
+	w.storageWriter = nil
+	if err != nil {
+		w.Abort()
+		return err
+	}
+
+	// Create the marker object.
+	if w.p != nil {
+		if err := w.p.sharedCreateRef(w.meta); err != nil {
+			w.Abort()
+			return err
+		}
+	}
+	return nil
+}
+
+// Abort is part of the Writable interface.
+func (w *sharedWritable) Abort() {
+	if w.storageWriter != nil {
+		_ = w.storageWriter.Close()
+		w.storageWriter = nil
+	}
+	if w.p != nil {
+		w.p.removeMetadata(w.meta.DiskFileNum)
+	}
+	// TODO(radu): delete the object if it was created.
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache/shared_cache.go

@@ -0,0 +1,900 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sharedcache
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math/bits"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+	"github.com/cockroachdb/pebble/vfs"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Exported to enable exporting from package pebble to enable
+// exporting metrics with below buckets in CRDB.
+var (
+	IOBuckets           = prometheus.ExponentialBucketsRange(float64(time.Millisecond*1), float64(10*time.Second), 50)
+	ChannelWriteBuckets = prometheus.ExponentialBucketsRange(float64(time.Microsecond*1), float64(10*time.Second), 50)
+)
+
+// Cache is a persistent cache backed by a local filesystem. It is intended
+// to cache data that is in slower shared storage (e.g. S3), hence the
+// package name 'sharedcache'.
+type Cache struct {
+	shards       []shard
+	writeWorkers writeWorkers
+
+	bm                blockMath
+	shardingBlockSize int64
+
+	logger  base.Logger
+	metrics internalMetrics
+}
+
+// Metrics is a struct containing metrics exported by the secondary cache.
+// TODO(josh): Reconsider the set of metrics exported by the secondary cache
+// before we release the secondary cache to users. We choose to export many metrics
+// right now, so we learn a lot from the benchmarking we are doing over the 23.2
+// cycle.
+type Metrics struct {
+	// The number of sstable bytes stored in the cache.
+	Size int64
+	// The count of cache blocks in the cache (not sstable blocks).
+	Count int64
+
+	// The number of calls to ReadAt.
+	TotalReads int64
+	// The number of calls to ReadAt that require reading data from 2+ shards.
+	MultiShardReads int64
+	// The number of calls to ReadAt that require reading data from 2+ cache blocks.
+	MultiBlockReads int64
+	// The number of calls to ReadAt where all data returned was read from the cache.
+	ReadsWithFullHit int64
+	// The number of calls to ReadAt where some data returned was read from the cache.
+	ReadsWithPartialHit int64
+	// The number of calls to ReadAt where no data returned was read from the cache.
+	ReadsWithNoHit int64
+
+	// The number of times a cache block was evicted from the cache.
+	Evictions int64
+	// The number of times writing a cache block to the cache failed.
+	WriteBackFailures int64
+
+	// The latency of calls to get some data from the cache.
+	GetLatency prometheus.Histogram
+	// The latency of reads of a single cache block from disk.
+	DiskReadLatency prometheus.Histogram
+	// The latency of writing data to write back to the cache to a channel.
+	// Generally should be low, but if the channel is full, could be high.
+	QueuePutLatency prometheus.Histogram
+	// The latency of calls to put some data read from block storage into the cache.
+	PutLatency prometheus.Histogram
+	// The latency of writes of a single cache block to disk.
+	DiskWriteLatency prometheus.Histogram
+}
+
+// See docs at Metrics.
+type internalMetrics struct {
+	count atomic.Int64
+
+	totalReads          atomic.Int64
+	multiShardReads     atomic.Int64
+	multiBlockReads     atomic.Int64
+	readsWithFullHit    atomic.Int64
+	readsWithPartialHit atomic.Int64
+	readsWithNoHit      atomic.Int64
+
+	evictions         atomic.Int64
+	writeBackFailures atomic.Int64
+
+	getLatency       prometheus.Histogram
+	diskReadLatency  prometheus.Histogram
+	queuePutLatency  prometheus.Histogram
+	putLatency       prometheus.Histogram
+	diskWriteLatency prometheus.Histogram
+}
+
+const (
+	// writeWorkersPerShard is used to establish the number of worker goroutines
+	// that perform writes to the cache.
+	writeWorkersPerShard = 4
+	// writeTaskPerWorker is used to establish how many tasks can be queued up
+	// until we have to block.
+	writeTasksPerWorker = 4
+)
+
+// Open opens a cache. If there is no existing cache at fsDir, a new one
+// is created.
+func Open(
+	fs vfs.FS,
+	logger base.Logger,
+	fsDir string,
+	blockSize int,
+	// shardingBlockSize is the size of a shard block. The cache is split into contiguous
+	// shardingBlockSize units. The units are distributed across multiple independent shards
+	// of the cache, via a hash(offset) modulo num shards operation. The cache replacement
+	// policies operate at the level of shard, not whole cache. This is done to reduce lock
+	// contention.
+	shardingBlockSize int64,
+	sizeBytes int64,
+	numShards int,
+) (*Cache, error) {
+	if minSize := shardingBlockSize * int64(numShards); sizeBytes < minSize {
+		// Up the size so that we have one block per shard. In practice, this should
+		// only happen in tests.
+		sizeBytes = minSize
+	}
+
+	c := &Cache{
+		logger:            logger,
+		bm:                makeBlockMath(blockSize),
+		shardingBlockSize: shardingBlockSize,
+	}
+	c.shards = make([]shard, numShards)
+	blocksPerShard := sizeBytes / int64(numShards) / int64(blockSize)
+	for i := range c.shards {
+		if err := c.shards[i].init(c, fs, fsDir, i, blocksPerShard, blockSize, shardingBlockSize); err != nil {
+			return nil, err
+		}
+	}
+
+	c.writeWorkers.Start(c, numShards*writeWorkersPerShard)
+
+	c.metrics.getLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
+	c.metrics.diskReadLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
+	c.metrics.putLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
+	c.metrics.diskWriteLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
+
+	// Measures a channel write, so lower min.
+	c.metrics.queuePutLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: ChannelWriteBuckets})
+
+	return c, nil
+}
+
+// Close closes the cache. Methods such as ReadAt should not be called after Close is
+// called.
+func (c *Cache) Close() error {
+	c.writeWorkers.Stop()
+
+	var retErr error
+	for i := range c.shards {
+		if err := c.shards[i].close(); err != nil && retErr == nil {
+			retErr = err
+		}
+	}
+	c.shards = nil
+	return retErr
+}
+
+// Metrics return metrics for the cache. Callers should not mutate
+// the returned histograms, which are pointer types.
+func (c *Cache) Metrics() Metrics {
+	return Metrics{
+		Count:               c.metrics.count.Load(),
+		Size:                c.metrics.count.Load() * int64(c.bm.BlockSize()),
+		TotalReads:          c.metrics.totalReads.Load(),
+		MultiShardReads:     c.metrics.multiShardReads.Load(),
+		MultiBlockReads:     c.metrics.multiBlockReads.Load(),
+		ReadsWithFullHit:    c.metrics.readsWithFullHit.Load(),
+		ReadsWithPartialHit: c.metrics.readsWithPartialHit.Load(),
+		ReadsWithNoHit:      c.metrics.readsWithNoHit.Load(),
+		Evictions:           c.metrics.evictions.Load(),
+		WriteBackFailures:   c.metrics.writeBackFailures.Load(),
+		GetLatency:          c.metrics.getLatency,
+		DiskReadLatency:     c.metrics.diskReadLatency,
+		QueuePutLatency:     c.metrics.queuePutLatency,
+		PutLatency:          c.metrics.putLatency,
+		DiskWriteLatency:    c.metrics.diskWriteLatency,
+	}
+}
+
+// ReadFlags contains options for Cache.ReadAt.
+type ReadFlags struct {
+	// ReadOnly instructs ReadAt to not write any new data into the cache; it is
+	// used when the data is unlikely to be used again.
+	ReadOnly bool
+}
+
+// ReadAt performs a read form an object, attempting to use cached data when
+// possible.
+func (c *Cache) ReadAt(
+	ctx context.Context,
+	fileNum base.DiskFileNum,
+	p []byte,
+	ofs int64,
+	objReader remote.ObjectReader,
+	objSize int64,
+	flags ReadFlags,
+) error {
+	c.metrics.totalReads.Add(1)
+	if ofs >= objSize {
+		if invariants.Enabled {
+			panic(fmt.Sprintf("invalid ReadAt offset %v %v", ofs, objSize))
+		}
+		return io.EOF
+	}
+	// TODO(radu): for compaction reads, we may not want to read from the cache at
+	// all.
+	{
+		start := time.Now()
+		n, err := c.get(fileNum, p, ofs)
+		c.metrics.getLatency.Observe(float64(time.Since(start)))
+		if err != nil {
+			return err
+		}
+		if n == len(p) {
+			// Everything was in cache!
+			c.metrics.readsWithFullHit.Add(1)
+			return nil
+		}
+		if n == 0 {
+			c.metrics.readsWithNoHit.Add(1)
+		} else {
+			c.metrics.readsWithPartialHit.Add(1)
+		}
+
+		// Note this. The below code does not need the original ofs, as with the earlier
+		// reading from the cache done, the relevant offset is ofs + int64(n). Same with p.
+		ofs += int64(n)
+		p = p[n:]
+
+		if invariants.Enabled {
+			if n != 0 && c.bm.Remainder(ofs) != 0 {
+				panic(fmt.Sprintf("after non-zero read from cache, ofs is not block-aligned: %v %v", ofs, n))
+			}
+		}
+	}
+
+	if flags.ReadOnly {
+		return objReader.ReadAt(ctx, p, ofs)
+	}
+
+	// We must do reads with offset & size that are multiples of the block size. Else
+	// later cache hits may return incorrect zeroed results from the cache.
+	firstBlockInd := c.bm.Block(ofs)
+	adjustedOfs := c.bm.BlockOffset(firstBlockInd)
+
+	// Take the length of what is left to read plus the length of the adjustment of
+	// the offset plus the size of a block minus one and divide by the size of a block
+	// to get the number of blocks to read from the object.
+	sizeOfOffAdjustment := int(ofs - adjustedOfs)
+	adjustedLen := int(c.bm.RoundUp(int64(len(p) + sizeOfOffAdjustment)))
+	adjustedP := make([]byte, adjustedLen)
+
+	// Read the rest from the object. We may need to cap the length to avoid past EOF reads.
+	eofCap := int64(adjustedLen)
+	if adjustedOfs+eofCap > objSize {
+		eofCap = objSize - adjustedOfs
+	}
+	if err := objReader.ReadAt(ctx, adjustedP[:eofCap], adjustedOfs); err != nil {
+		return err
+	}
+	copy(p, adjustedP[sizeOfOffAdjustment:])
+
+	start := time.Now()
+	c.writeWorkers.QueueWrite(fileNum, adjustedP, adjustedOfs)
+	c.metrics.queuePutLatency.Observe(float64(time.Since(start)))
+
+	return nil
+}
+
+// get attempts to read the requested data from the cache, if it is already
+// there.
+//
+// If all data is available, returns n = len(p).
+//
+// If data is partially available, a prefix of the data is read; returns n < len(p)
+// and no error. If no prefix is available, returns n = 0 and no error.
+func (c *Cache) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ error) {
+	// The data extent might cross shard boundaries, hence the loop. In the hot
+	// path, max two iterations of this loop will be executed, since reads are sized
+	// in units of sstable block size.
+	var multiShard bool
+	for {
+		shard := c.getShard(fileNum, ofs+int64(n))
+		cappedLen := len(p[n:])
+		if toBoundary := int(c.shardingBlockSize - ((ofs + int64(n)) % c.shardingBlockSize)); cappedLen > toBoundary {
+			cappedLen = toBoundary
+		}
+		numRead, err := shard.get(fileNum, p[n:n+cappedLen], ofs+int64(n))
+		if err != nil {
+			return n, err
+		}
+		n += numRead
+		if numRead < cappedLen {
+			// We only read a prefix from this shard.
+			return n, nil
+		}
+		if n == len(p) {
+			// We are done.
+			return n, nil
+		}
+		// Data extent crosses shard boundary, continue with next shard.
+		if !multiShard {
+			c.metrics.multiShardReads.Add(1)
+			multiShard = true
+		}
+	}
+}
+
+// set attempts to write the requested data to the cache. Both ofs & len(p) must
+// be multiples of the block size.
+//
+// If all of p is not written to the shard, set returns a non-nil error.
+func (c *Cache) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
+	if invariants.Enabled {
+		if c.bm.Remainder(ofs) != 0 || c.bm.Remainder(int64(len(p))) != 0 {
+			panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p)))
+		}
+	}
+
+	// The data extent might cross shard boundaries, hence the loop. In the hot
+	// path, max two iterations of this loop will be executed, since reads are sized
+	// in units of sstable block size.
+	n := 0
+	for {
+		shard := c.getShard(fileNum, ofs+int64(n))
+		cappedLen := len(p[n:])
+		if toBoundary := int(c.shardingBlockSize - ((ofs + int64(n)) % c.shardingBlockSize)); cappedLen > toBoundary {
+			cappedLen = toBoundary
+		}
+		err := shard.set(fileNum, p[n:n+cappedLen], ofs+int64(n))
+		if err != nil {
+			return err
+		}
+		// set returns an error if cappedLen bytes aren't written to the shard.
+		n += cappedLen
+		if n == len(p) {
+			// We are done.
+			return nil
+		}
+		// Data extent crosses shard boundary, continue with next shard.
+	}
+}
+
+func (c *Cache) getShard(fileNum base.DiskFileNum, ofs int64) *shard {
+	const prime64 = 1099511628211
+	hash := uint64(fileNum.FileNum())*prime64 + uint64(ofs/c.shardingBlockSize)
+	// TODO(josh): Instance change ops are often run in production. Such an operation
+	// updates len(c.shards); see openSharedCache. As a result, the behavior of this
+	// function changes, and the cache empties out at restart time. We may want a better
+	// story here eventually.
+	return &c.shards[hash%uint64(len(c.shards))]
+}
+
+type shard struct {
+	cache             *Cache
+	file              vfs.File
+	sizeInBlocks      int64
+	bm                blockMath
+	shardingBlockSize int64
+	mu                struct {
+		sync.Mutex
+		// TODO(josh): None of these datastructures are space-efficient.
+		// Focusing on correctness to start.
+		where  whereMap
+		blocks []cacheBlockState
+		// Head of LRU list (doubly-linked circular).
+		lruHead cacheBlockIndex
+		// Head of free list (singly-linked chain).
+		freeHead cacheBlockIndex
+	}
+}
+
+type cacheBlockState struct {
+	lock    lockState
+	logical logicalBlockID
+
+	// next is the next block in the LRU or free list (or invalidBlockIndex if it
+	// is the last block in the free list).
+	next cacheBlockIndex
+
+	// prev is the previous block in the LRU list. It is not used when the block
+	// is in the free list.
+	prev cacheBlockIndex
+}
+
+// Maps a logical block in an SST to an index of the cache block with the
+// file contents (to the "cache block index").
+type whereMap map[logicalBlockID]cacheBlockIndex
+
+type logicalBlockID struct {
+	filenum       base.DiskFileNum
+	cacheBlockIdx cacheBlockIndex
+}
+
+type lockState int64
+
+const (
+	unlocked lockState = 0
+	// >0 lockState tracks the number of distinct readers of some cache block / logical block
+	// which is in the secondary cache. It is used to ensure that a cache block is not evicted
+	// and overwritten, while there are active readers.
+	readLockTakenInc = 1
+	// -1 lockState indicates that some cache block is currently being populated with data from
+	// blob storage. It is used to ensure that a cache block is not read or evicted again, while
+	// it is being populated.
+	writeLockTaken = -1
+)
+
+func (s *shard) init(
+	cache *Cache,
+	fs vfs.FS,
+	fsDir string,
+	shardIdx int,
+	sizeInBlocks int64,
+	blockSize int,
+	shardingBlockSize int64,
+) error {
+	*s = shard{
+		cache:        cache,
+		sizeInBlocks: sizeInBlocks,
+	}
+	if blockSize < 1024 || shardingBlockSize%int64(blockSize) != 0 {
+		return errors.Newf("invalid block size %d (must divide %d)", blockSize, shardingBlockSize)
+	}
+	s.bm = makeBlockMath(blockSize)
+	s.shardingBlockSize = shardingBlockSize
+	file, err := fs.OpenReadWrite(fs.PathJoin(fsDir, fmt.Sprintf("SHARED-CACHE-%03d", shardIdx)))
+	if err != nil {
+		return err
+	}
+	// TODO(radu): truncate file if necessary (especially important if we restart
+	// with more shards).
+	if err := file.Preallocate(0, int64(blockSize)*sizeInBlocks); err != nil {
+		return err
+	}
+	s.file = file
+
+	// TODO(josh): Right now, the secondary cache is not persistent. All existing
+	// cache contents will be over-written, since all metadata is only stored in
+	// memory.
+	s.mu.where = make(whereMap)
+	s.mu.blocks = make([]cacheBlockState, sizeInBlocks)
+	s.mu.lruHead = invalidBlockIndex
+	s.mu.freeHead = invalidBlockIndex
+	for i := range s.mu.blocks {
+		s.freePush(cacheBlockIndex(i))
+	}
+
+	return nil
+}
+
+func (s *shard) close() error {
+	defer func() {
+		s.file = nil
+	}()
+	return s.file.Close()
+}
+
+// freePush pushes a block to the front of the free list.
+func (s *shard) freePush(index cacheBlockIndex) {
+	s.mu.blocks[index].next = s.mu.freeHead
+	s.mu.freeHead = index
+}
+
+// freePop removes the block from the front of the free list. Must not be called
+// if the list is empty (i.e. freeHead = invalidBlockIndex).
+func (s *shard) freePop() cacheBlockIndex {
+	index := s.mu.freeHead
+	s.mu.freeHead = s.mu.blocks[index].next
+	return index
+}
+
+// lruInsertFront inserts a block at the front of the LRU list.
+func (s *shard) lruInsertFront(index cacheBlockIndex) {
+	b := &s.mu.blocks[index]
+	if s.mu.lruHead == invalidBlockIndex {
+		b.next = index
+		b.prev = index
+	} else {
+		b.next = s.mu.lruHead
+		h := &s.mu.blocks[s.mu.lruHead]
+		b.prev = h.prev
+		s.mu.blocks[h.prev].next = index
+		h.prev = index
+	}
+	s.mu.lruHead = index
+}
+
+func (s *shard) lruNext(index cacheBlockIndex) cacheBlockIndex {
+	return s.mu.blocks[index].next
+}
+
+func (s *shard) lruPrev(index cacheBlockIndex) cacheBlockIndex {
+	return s.mu.blocks[index].prev
+}
+
+// lruUnlink removes a block from the LRU list.
+func (s *shard) lruUnlink(index cacheBlockIndex) {
+	b := &s.mu.blocks[index]
+	if b.next == index {
+		s.mu.lruHead = invalidBlockIndex
+	} else {
+		s.mu.blocks[b.prev].next = b.next
+		s.mu.blocks[b.next].prev = b.prev
+		if s.mu.lruHead == index {
+			s.mu.lruHead = b.next
+		}
+	}
+	b.next, b.prev = invalidBlockIndex, invalidBlockIndex
+}
+
+// get attempts to read the requested data from the shard. The data must not
+// cross a shard boundary.
+//
+// If all data is available, returns n = len(p).
+//
+// If data is partially available, a prefix of the data is read; returns n < len(p)
+// and no error. If no prefix is available, returns n = 0 and no error.
+//
+// TODO(josh): Today, if there are two cache blocks needed to satisfy a read, and the
+// first block is not in the cache and the second one is, we will read both from
+// blob storage. We should fix this. This is not an unlikely scenario if we are doing
+// a reverse scan, since those iterate over sstable blocks in reverse order and due to
+// cache block aligned reads will have read the suffix of the sstable block that will
+// be needed next.
+func (s *shard) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ error) {
+	if invariants.Enabled {
+		if ofs/s.shardingBlockSize != (ofs+int64(len(p))-1)/s.shardingBlockSize {
+			panic(fmt.Sprintf("get crosses shard boundary: %v %v", ofs, len(p)))
+		}
+		s.assertShardStateIsConsistent()
+	}
+
+	// The data extent might cross cache block boundaries, hence the loop. In the hot
+	// path, max two iterations of this loop will be executed, since reads are sized
+	// in units of sstable block size.
+	var multiBlock bool
+	for {
+		k := logicalBlockID{
+			filenum:       fileNum,
+			cacheBlockIdx: s.bm.Block(ofs + int64(n)),
+		}
+		s.mu.Lock()
+		cacheBlockIdx, ok := s.mu.where[k]
+		// TODO(josh): Multiple reads within the same few milliseconds (anything that is smaller
+		// than blob storage read latency) that miss on the same logical block ID will not necessarily
+		// be rare. We may want to do only one read, with the later readers blocking on the first read
+		// completing. This could be implemented either here or in the primary block cache. See
+		// https://github.com/cockroachdb/pebble/pull/2586 for additional discussion.
+		if !ok {
+			s.mu.Unlock()
+			return n, nil
+		}
+		if s.mu.blocks[cacheBlockIdx].lock == writeLockTaken {
+			// In practice, if we have two reads of the same SST block in close succession, we
+			// would expect the second to hit in the in-memory block cache. So it's not worth
+			// optimizing this case here.
+			s.mu.Unlock()
+			return n, nil
+		}
+		s.mu.blocks[cacheBlockIdx].lock += readLockTakenInc
+		// Move to front of the LRU list.
+		s.lruUnlink(cacheBlockIdx)
+		s.lruInsertFront(cacheBlockIdx)
+		s.mu.Unlock()
+
+		readAt := s.bm.BlockOffset(cacheBlockIdx)
+		readSize := s.bm.BlockSize()
+		if n == 0 { // if first read
+			rem := s.bm.Remainder(ofs)
+			readAt += rem
+			readSize -= int(rem)
+		}
+
+		if len(p[n:]) <= readSize {
+			start := time.Now()
+			numRead, err := s.file.ReadAt(p[n:], readAt)
+			s.cache.metrics.diskReadLatency.Observe(float64(time.Since(start)))
+			s.dropReadLock(cacheBlockIdx)
+			return n + numRead, err
+		}
+		start := time.Now()
+		numRead, err := s.file.ReadAt(p[n:n+readSize], readAt)
+		s.cache.metrics.diskReadLatency.Observe(float64(time.Since(start)))
+		s.dropReadLock(cacheBlockIdx)
+		if err != nil {
+			return 0, err
+		}
+
+		// Note that numRead == readSize, since we checked for an error above.
+		n += numRead
+
+		if !multiBlock {
+			s.cache.metrics.multiBlockReads.Add(1)
+			multiBlock = true
+		}
+	}
+}
+
+// set attempts to write the requested data to the shard. The data must not
+// cross a shard boundary, and both ofs & len(p) must be multiples of the
+// block size.
+//
+// If all of p is not written to the shard, set returns a non-nil error.
+func (s *shard) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
+	if invariants.Enabled {
+		if ofs/s.shardingBlockSize != (ofs+int64(len(p))-1)/s.shardingBlockSize {
+			panic(fmt.Sprintf("set crosses shard boundary: %v %v", ofs, len(p)))
+		}
+		if s.bm.Remainder(ofs) != 0 || s.bm.Remainder(int64(len(p))) != 0 {
+			panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p)))
+		}
+		s.assertShardStateIsConsistent()
+	}
+
+	// The data extent might cross cache block boundaries, hence the loop. In the hot
+	// path, max two iterations of this loop will be executed, since reads are sized
+	// in units of sstable block size.
+	n := 0
+	for {
+		if n == len(p) {
+			return nil
+		}
+		if invariants.Enabled {
+			if n > len(p) {
+				panic(fmt.Sprintf("set with n greater than len(p): %v %v", n, len(p)))
+			}
+		}
+
+		// If the logical block is already in the cache, we should skip doing a set.
+		k := logicalBlockID{
+			filenum:       fileNum,
+			cacheBlockIdx: s.bm.Block(ofs + int64(n)),
+		}
+		s.mu.Lock()
+		if _, ok := s.mu.where[k]; ok {
+			s.mu.Unlock()
+			n += s.bm.BlockSize()
+			continue
+		}
+
+		var cacheBlockIdx cacheBlockIndex
+		if s.mu.freeHead == invalidBlockIndex {
+			if invariants.Enabled && s.mu.lruHead == invalidBlockIndex {
+				panic("both LRU and free lists empty")
+			}
+
+			// Find the last element in the LRU list which is not locked.
+			for idx := s.lruPrev(s.mu.lruHead); ; idx = s.lruPrev(idx) {
+				if lock := s.mu.blocks[idx].lock; lock == unlocked {
+					cacheBlockIdx = idx
+					break
+				}
+				if idx == s.mu.lruHead {
+					// No unlocked block to evict.
+					//
+					// TODO(josh): We may want to block until a block frees up, instead of returning
+					// an error here. But I think we can do that later on, e.g. after running some production
+					// experiments.
+					s.mu.Unlock()
+					return errors.New("no block to evict so skipping write to cache")
+				}
+			}
+			s.cache.metrics.evictions.Add(1)
+			s.lruUnlink(cacheBlockIdx)
+			delete(s.mu.where, s.mu.blocks[cacheBlockIdx].logical)
+		} else {
+			s.cache.metrics.count.Add(1)
+			cacheBlockIdx = s.freePop()
+		}
+
+		s.lruInsertFront(cacheBlockIdx)
+		s.mu.where[k] = cacheBlockIdx
+		s.mu.blocks[cacheBlockIdx].logical = k
+		s.mu.blocks[cacheBlockIdx].lock = writeLockTaken
+		s.mu.Unlock()
+
+		writeAt := s.bm.BlockOffset(cacheBlockIdx)
+
+		writeSize := s.bm.BlockSize()
+		if len(p[n:]) <= writeSize {
+			writeSize = len(p[n:])
+		}
+
+		start := time.Now()
+		_, err := s.file.WriteAt(p[n:n+writeSize], writeAt)
+		s.cache.metrics.diskWriteLatency.Observe(float64(time.Since(start)))
+		if err != nil {
+			// Free the block.
+			s.mu.Lock()
+			defer s.mu.Unlock()
+
+			delete(s.mu.where, k)
+			s.lruUnlink(cacheBlockIdx)
+			s.freePush(cacheBlockIdx)
+			return err
+		}
+		s.dropWriteLock(cacheBlockIdx)
+		n += writeSize
+	}
+}
+
+// Doesn't inline currently. This might be okay, but something to keep in mind.
+func (s *shard) dropReadLock(cacheBlockInd cacheBlockIndex) {
+	s.mu.Lock()
+	s.mu.blocks[cacheBlockInd].lock -= readLockTakenInc
+	if invariants.Enabled && s.mu.blocks[cacheBlockInd].lock < 0 {
+		panic(fmt.Sprintf("unexpected lock state %v in dropReadLock", s.mu.blocks[cacheBlockInd].lock))
+	}
+	s.mu.Unlock()
+}
+
+// Doesn't inline currently. This might be okay, but something to keep in mind.
+func (s *shard) dropWriteLock(cacheBlockInd cacheBlockIndex) {
+	s.mu.Lock()
+	if invariants.Enabled && s.mu.blocks[cacheBlockInd].lock != writeLockTaken {
+		panic(fmt.Sprintf("unexpected lock state %v in dropWriteLock", s.mu.blocks[cacheBlockInd].lock))
+	}
+	s.mu.blocks[cacheBlockInd].lock = unlocked
+	s.mu.Unlock()
+}
+
+func (s *shard) assertShardStateIsConsistent() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	lruLen := 0
+	if s.mu.lruHead != invalidBlockIndex {
+		for b := s.mu.lruHead; ; {
+			lruLen++
+			if idx, ok := s.mu.where[s.mu.blocks[b].logical]; !ok || idx != b {
+				panic("block in LRU list with no entry in where map")
+			}
+			b = s.lruNext(b)
+			if b == s.mu.lruHead {
+				break
+			}
+		}
+	}
+	if lruLen != len(s.mu.where) {
+		panic(fmt.Sprintf("lru list len is %d but where map has %d entries", lruLen, len(s.mu.where)))
+	}
+	freeLen := 0
+	for n := s.mu.freeHead; n != invalidBlockIndex; n = s.mu.blocks[n].next {
+		freeLen++
+	}
+
+	if lruLen+freeLen != int(s.sizeInBlocks) {
+		panic(fmt.Sprintf("%d lru blocks and %d free blocks don't add up to %d", lruLen, freeLen, s.sizeInBlocks))
+	}
+	for i := range s.mu.blocks {
+		if state := s.mu.blocks[i].lock; state < writeLockTaken {
+			panic(fmt.Sprintf("lock state %v is not allowed", state))
+		}
+	}
+}
+
+// cacheBlockIndex is the index of a blockSize-aligned cache block.
+type cacheBlockIndex int64
+
+// invalidBlockIndex is used for the head of a list when the list is empty.
+const invalidBlockIndex cacheBlockIndex = -1
+
+// blockMath is a helper type for performing conversions between offsets and
+// block indexes.
+type blockMath struct {
+	blockSizeBits int8
+}
+
+func makeBlockMath(blockSize int) blockMath {
+	bm := blockMath{
+		blockSizeBits: int8(bits.Len64(uint64(blockSize)) - 1),
+	}
+	if blockSize != (1 << bm.blockSizeBits) {
+		panic(fmt.Sprintf("blockSize %d is not a power of 2", blockSize))
+	}
+	return bm
+}
+
+func (bm blockMath) mask() int64 {
+	return (1 << bm.blockSizeBits) - 1
+}
+
+// BlockSize returns the block size.
+func (bm blockMath) BlockSize() int {
+	return 1 << bm.blockSizeBits
+}
+
+// Block returns the block index containing the given offset.
+func (bm blockMath) Block(offset int64) cacheBlockIndex {
+	return cacheBlockIndex(offset >> bm.blockSizeBits)
+}
+
+// Remainder returns the offset relative to the start of the cache block.
+func (bm blockMath) Remainder(offset int64) int64 {
+	return offset & bm.mask()
+}
+
+// BlockOffset returns the object offset where the given block starts.
+func (bm blockMath) BlockOffset(block cacheBlockIndex) int64 {
+	return int64(block) << bm.blockSizeBits
+}
+
+// RoundUp rounds up the given value to the closest multiple of block size.
+func (bm blockMath) RoundUp(x int64) int64 {
+	return (x + bm.mask()) & ^(bm.mask())
+}
+
+type writeWorkers struct {
+	doneCh        chan struct{}
+	doneWaitGroup sync.WaitGroup
+
+	numWorkers int
+	tasksCh    chan writeTask
+}
+
+type writeTask struct {
+	fileNum base.DiskFileNum
+	p       []byte
+	offset  int64
+}
+
+// Start starts the worker goroutines.
+func (w *writeWorkers) Start(c *Cache, numWorkers int) {
+	doneCh := make(chan struct{})
+	tasksCh := make(chan writeTask, numWorkers*writeTasksPerWorker)
+
+	w.numWorkers = numWorkers
+	w.doneCh = doneCh
+	w.tasksCh = tasksCh
+	w.doneWaitGroup.Add(numWorkers)
+	for i := 0; i < numWorkers; i++ {
+		go func() {
+			defer w.doneWaitGroup.Done()
+			for {
+				select {
+				case <-doneCh:
+					return
+				case task, ok := <-tasksCh:
+					if !ok {
+						// The tasks channel was closed; this is used in testing code to
+						// ensure all writes are completed.
+						return
+					}
+					// TODO(radu): set() can perform multiple writes; perhaps each one
+					// should be its own task.
+					start := time.Now()
+					err := c.set(task.fileNum, task.p, task.offset)
+					c.metrics.putLatency.Observe(float64(time.Since(start)))
+					if err != nil {
+						c.metrics.writeBackFailures.Add(1)
+						// TODO(radu): throttle logs.
+						c.logger.Infof("writing back to cache after miss failed: %v", err)
+					}
+				}
+			}
+		}()
+	}
+}
+
+// Stop waits for any in-progress writes to complete and stops the worker
+// goroutines and waits for any in-pro. Any queued writes not yet started are
+// discarded.
+func (w *writeWorkers) Stop() {
+	close(w.doneCh)
+	w.doneCh = nil
+	w.tasksCh = nil
+	w.doneWaitGroup.Wait()
+}
+
+// QueueWrite adds a write task to the queue. Can block if the queue is full.
+func (w *writeWorkers) QueueWrite(fileNum base.DiskFileNum, p []byte, offset int64) {
+	w.tasksCh <- writeTask{
+		fileNum: fileNum,
+		p:       p,
+		offset:  offset,
+	}
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs.go

@@ -0,0 +1,113 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+func (p *provider) vfsPath(fileType base.FileType, fileNum base.DiskFileNum) string {
+	return base.MakeFilepath(p.st.FS, p.st.FSDirName, fileType, fileNum)
+}
+
+func (p *provider) vfsOpenForReading(
+	ctx context.Context,
+	fileType base.FileType,
+	fileNum base.DiskFileNum,
+	opts objstorage.OpenOptions,
+) (objstorage.Readable, error) {
+	filename := p.vfsPath(fileType, fileNum)
+	file, err := p.st.FS.Open(filename, vfs.RandomReadsOption)
+	if err != nil {
+		if opts.MustExist {
+			base.MustExist(p.st.FS, filename, p.st.Logger, err)
+		}
+		return nil, err
+	}
+	readaheadConfig := DefaultReadaheadConfig
+	if f := p.st.Local.ReadaheadConfigFn; f != nil {
+		readaheadConfig = f()
+	}
+	return newFileReadable(file, p.st.FS, readaheadConfig, filename)
+}
+
+func (p *provider) vfsCreate(
+	_ context.Context, fileType base.FileType, fileNum base.DiskFileNum,
+) (objstorage.Writable, objstorage.ObjectMetadata, error) {
+	filename := p.vfsPath(fileType, fileNum)
+	file, err := p.st.FS.Create(filename)
+	if err != nil {
+		return nil, objstorage.ObjectMetadata{}, err
+	}
+	file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{
+		NoSyncOnClose: p.st.NoSyncOnClose,
+		BytesPerSync:  p.st.BytesPerSync,
+	})
+	meta := objstorage.ObjectMetadata{
+		DiskFileNum: fileNum,
+		FileType:    fileType,
+	}
+	return newFileBufferedWritable(file), meta, nil
+}
+
+func (p *provider) vfsRemove(fileType base.FileType, fileNum base.DiskFileNum) error {
+	return p.st.FSCleaner.Clean(p.st.FS, fileType, p.vfsPath(fileType, fileNum))
+}
+
+// vfsInit finds any local FS objects.
+func (p *provider) vfsInit() error {
+	listing := p.st.FSDirInitialListing
+	if listing == nil {
+		var err error
+		listing, err = p.st.FS.List(p.st.FSDirName)
+		if err != nil {
+			return errors.Wrapf(err, "pebble: could not list store directory")
+		}
+	}
+
+	for _, filename := range listing {
+		fileType, fileNum, ok := base.ParseFilename(p.st.FS, filename)
+		if ok && fileType == base.FileTypeTable {
+			o := objstorage.ObjectMetadata{
+				FileType:    fileType,
+				DiskFileNum: fileNum,
+			}
+			p.mu.knownObjects[o.DiskFileNum] = o
+		}
+	}
+	return nil
+}
+
+func (p *provider) vfsSync() error {
+	p.mu.Lock()
+	shouldSync := p.mu.localObjectsChanged
+	p.mu.localObjectsChanged = false
+	p.mu.Unlock()
+
+	if !shouldSync {
+		return nil
+	}
+	if err := p.fsDir.Sync(); err != nil {
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		p.mu.localObjectsChanged = true
+		return err
+	}
+	return nil
+}
+
+func (p *provider) vfsSize(fileType base.FileType, fileNum base.DiskFileNum) (int64, error) {
+	filename := p.vfsPath(fileType, fileNum)
+	stat, err := p.st.FS.Stat(filename)
+	if err != nil {
+		return 0, err
+	}
+	return stat.Size(), nil
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go

@@ -0,0 +1,236 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+const fileMaxReadaheadSize = 256 * 1024 /* 256KB */
+
+// fileReadable implements objstorage.Readable on top of a vfs.File.
+//
+// The implementation might use Prealloc and might reopen the file with
+// SequentialReadsOption.
+type fileReadable struct {
+	file vfs.File
+	size int64
+
+	// The following fields are used to possibly open the file again using the
+	// sequential reads option (see vfsReadHandle).
+	filename        string
+	fs              vfs.FS
+	readaheadConfig ReadaheadConfig
+}
+
+var _ objstorage.Readable = (*fileReadable)(nil)
+
+func newFileReadable(
+	file vfs.File, fs vfs.FS, readaheadConfig ReadaheadConfig, filename string,
+) (*fileReadable, error) {
+	info, err := file.Stat()
+	if err != nil {
+		return nil, err
+	}
+	r := &fileReadable{
+		file:            file,
+		size:            info.Size(),
+		filename:        filename,
+		fs:              fs,
+		readaheadConfig: readaheadConfig,
+	}
+	invariants.SetFinalizer(r, func(obj interface{}) {
+		if obj.(*fileReadable).file != nil {
+			fmt.Fprintf(os.Stderr, "Readable was not closed")
+			os.Exit(1)
+		}
+	})
+	return r, nil
+}
+
+// ReadAt is part of the objstorage.Readable interface.
+func (r *fileReadable) ReadAt(_ context.Context, p []byte, off int64) error {
+	n, err := r.file.ReadAt(p, off)
+	if invariants.Enabled && err == nil && n != len(p) {
+		panic("short read")
+	}
+	return err
+}
+
+// Close is part of the objstorage.Readable interface.
+func (r *fileReadable) Close() error {
+	defer func() { r.file = nil }()
+	return r.file.Close()
+}
+
+// Size is part of the objstorage.Readable interface.
+func (r *fileReadable) Size() int64 {
+	return r.size
+}
+
+// NewReadHandle is part of the objstorage.Readable interface.
+func (r *fileReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
+	rh := readHandlePool.Get().(*vfsReadHandle)
+	rh.init(r)
+	return rh
+}
+
+type vfsReadHandle struct {
+	r             *fileReadable
+	rs            readaheadState
+	readaheadMode ReadaheadMode
+
+	// sequentialFile holds a file descriptor to the same underlying File,
+	// except with fadvise(FADV_SEQUENTIAL) called on it to take advantage of
+	// OS-level readahead. Once this is non-nil, the other variables in
+	// readaheadState don't matter much as we defer to OS-level readahead.
+	sequentialFile vfs.File
+}
+
+var _ objstorage.ReadHandle = (*vfsReadHandle)(nil)
+
+var readHandlePool = sync.Pool{
+	New: func() interface{} {
+		i := &vfsReadHandle{}
+		// Note: this is a no-op if invariants are disabled or race is enabled.
+		invariants.SetFinalizer(i, func(obj interface{}) {
+			if obj.(*vfsReadHandle).r != nil {
+				fmt.Fprintf(os.Stderr, "ReadHandle was not closed")
+				os.Exit(1)
+			}
+		})
+		return i
+	},
+}
+
+func (rh *vfsReadHandle) init(r *fileReadable) {
+	*rh = vfsReadHandle{
+		r:             r,
+		rs:            makeReadaheadState(fileMaxReadaheadSize),
+		readaheadMode: r.readaheadConfig.Speculative,
+	}
+}
+
+// Close is part of the objstorage.ReadHandle interface.
+func (rh *vfsReadHandle) Close() error {
+	var err error
+	if rh.sequentialFile != nil {
+		err = rh.sequentialFile.Close()
+	}
+	*rh = vfsReadHandle{}
+	readHandlePool.Put(rh)
+	return err
+}
+
+// ReadAt is part of the objstorage.ReadHandle interface.
+func (rh *vfsReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error {
+	if rh.sequentialFile != nil {
+		// Use OS-level read-ahead.
+		n, err := rh.sequentialFile.ReadAt(p, offset)
+		if invariants.Enabled && err == nil && n != len(p) {
+			panic("short read")
+		}
+		return err
+	}
+	if rh.readaheadMode != NoReadahead {
+		if readaheadSize := rh.rs.maybeReadahead(offset, int64(len(p))); readaheadSize > 0 {
+			if rh.readaheadMode == FadviseSequential && readaheadSize >= fileMaxReadaheadSize {
+				// We've reached the maximum readahead size. Beyond this point, rely on
+				// OS-level readahead.
+				rh.switchToOSReadahead()
+			} else {
+				_ = rh.r.file.Prefetch(offset, readaheadSize)
+			}
+		}
+	}
+	n, err := rh.r.file.ReadAt(p, offset)
+	if invariants.Enabled && err == nil && n != len(p) {
+		panic("short read")
+	}
+	return err
+}
+
+// SetupForCompaction is part of the objstorage.ReadHandle interface.
+func (rh *vfsReadHandle) SetupForCompaction() {
+	rh.readaheadMode = rh.r.readaheadConfig.Informed
+	if rh.readaheadMode == FadviseSequential {
+		rh.switchToOSReadahead()
+	}
+}
+
+func (rh *vfsReadHandle) switchToOSReadahead() {
+	if invariants.Enabled && rh.readaheadMode != FadviseSequential {
+		panic("readheadMode not respected")
+	}
+	if rh.sequentialFile != nil {
+		return
+	}
+
+	// TODO(radu): we could share the reopened file descriptor across multiple
+	// handles.
+	f, err := rh.r.fs.Open(rh.r.filename, vfs.SequentialReadsOption)
+	if err == nil {
+		rh.sequentialFile = f
+	}
+}
+
+// RecordCacheHit is part of the objstorage.ReadHandle interface.
+func (rh *vfsReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
+	if rh.sequentialFile != nil || rh.readaheadMode == NoReadahead {
+		// Using OS-level or no readahead, so do nothing.
+		return
+	}
+	rh.rs.recordCacheHit(offset, size)
+}
+
+// TestingCheckMaxReadahead returns true if the ReadHandle has switched to
+// OS-level read-ahead.
+func TestingCheckMaxReadahead(rh objstorage.ReadHandle) bool {
+	switch rh := rh.(type) {
+	case *vfsReadHandle:
+		return rh.sequentialFile != nil
+	case *PreallocatedReadHandle:
+		return rh.sequentialFile != nil
+	default:
+		panic("unknown ReadHandle type")
+	}
+}
+
+// PreallocatedReadHandle is used to avoid an allocation in NewReadHandle; see
+// UsePreallocatedReadHandle.
+type PreallocatedReadHandle struct {
+	vfsReadHandle
+}
+
+// Close is part of the objstorage.ReadHandle interface.
+func (rh *PreallocatedReadHandle) Close() error {
+	var err error
+	if rh.sequentialFile != nil {
+		err = rh.sequentialFile.Close()
+	}
+	rh.vfsReadHandle = vfsReadHandle{}
+	return err
+}
+
+// UsePreallocatedReadHandle is equivalent to calling readable.NewReadHandle()
+// but uses the existing storage of a PreallocatedReadHandle when possible
+// (currently this happens if we are reading from a local file).
+// The returned handle still needs to be closed.
+func UsePreallocatedReadHandle(
+	ctx context.Context, readable objstorage.Readable, rh *PreallocatedReadHandle,
+) objstorage.ReadHandle {
+	if r, ok := readable.(*fileReadable); ok {
+		rh.init(r)
+		return rh
+	}
+	return readable.NewReadHandle(ctx)
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_writable.go

@@ -0,0 +1,65 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"bufio"
+
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// NewFileWritable returns a Writable that uses a file as underlying storage.
+func NewFileWritable(file vfs.File) objstorage.Writable {
+	return newFileBufferedWritable(file)
+}
+
+type fileBufferedWritable struct {
+	file vfs.File
+	bw   *bufio.Writer
+}
+
+var _ objstorage.Writable = (*fileBufferedWritable)(nil)
+
+func newFileBufferedWritable(file vfs.File) *fileBufferedWritable {
+	return &fileBufferedWritable{
+		file: file,
+		bw:   bufio.NewWriter(file),
+	}
+}
+
+// Write is part of the objstorage.Writable interface.
+func (w *fileBufferedWritable) Write(p []byte) error {
+	// Ignoring the length written since bufio.Writer.Write is guaranteed to
+	// return an error if the length written is < len(p).
+	_, err := w.bw.Write(p)
+	return err
+}
+
+// Finish is part of the objstorage.Writable interface.
+func (w *fileBufferedWritable) Finish() error {
+	err := w.bw.Flush()
+	if err == nil {
+		err = w.file.Sync()
+	}
+	err = firstError(err, w.file.Close())
+	w.bw = nil
+	w.file = nil
+	return err
+}
+
+// Abort is part of the objstorage.Writable interface.
+func (w *fileBufferedWritable) Abort() {
+	_ = w.file.Close()
+	w.bw = nil
+	w.file = nil
+}
+
+func firstError(err0, err1 error) error {
+	if err0 != nil {
+		return err0
+	}
+	return err1
+}