Upgrade versions of vendored libs

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing.go

@@ -0,0 +1,68 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objiotracing
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// OpType indicates the type of operation.
+type OpType uint8
+
+// OpType values.
+const (
+	ReadOp OpType = iota
+	WriteOp
+	// RecordCacheHitOp happens when a read is satisfied from the block cache. See
+	// objstorage.ReadHandle.RecordCacheHit().
+	RecordCacheHitOp
+	// MaxReadaheadOp is a "meta operation" that configures a read handle for
+	// large sequential reads. See objstorage.ReadHandle.MaxReadahead().
+	MaxReadaheadOp
+)
+
+// Reason indicates the higher-level context of the operation.
+type Reason uint8
+
+// Reason values.
+const (
+	UnknownReason Reason = iota
+	ForFlush
+	ForCompaction
+	ForIngestion
+	// TODO(radu): add ForUserFacing.
+)
+
+// BlockType indicates the type of data block relevant to an operation.
+type BlockType uint8
+
+// BlockType values.
+const (
+	UnknownBlock BlockType = iota
+	DataBlock
+	ValueBlock
+	FilterBlock
+	MetadataBlock
+)
+
+// Event is the on-disk format of a tracing event. It is exported here so that
+// trace processing tools can use it by importing this package.
+type Event struct {
+	// Event start time as a Unix time (see time.Time.StartUnixNano()).
+	// Note that recorded events are not necessarily ordered by time - this is
+	// because separate event "streams" use local buffers (for performance).
+	StartUnixNano int64
+	Op            OpType
+	Reason        Reason
+	BlockType     BlockType
+	// LSM level plus one (with 0 indicating unknown level).
+	LevelPlusOne uint8
+	// Hardcoded padding so that struct layout doesn't depend on architecture.
+	_       uint32
+	FileNum base.FileNum
+	// HandleID is a unique identifier corresponding to an objstorage.ReadHandle;
+	// only set for read operations performed through a ReadHandle.
+	HandleID uint64
+	Offset   int64
+	Size     int64
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go

@@ -0,0 +1,59 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build !pebble_obj_io_tracing
+// +build !pebble_obj_io_tracing
+
+package objiotracing
+
+import (
+	"context"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// Enabled is used to short circuit tracing-related code in regular builds.
+const Enabled = false
+
+// Tracer manages the writing of object IO traces to files.
+type Tracer struct{}
+
+// Open creates a Tracer which generates trace files in the given directory.
+// Each trace file contains a series of Events (as they are in memory).
+func Open(fs vfs.FS, fsDir string) *Tracer {
+	return nil
+}
+
+// Close the tracer, flushing any remaining events.
+func (*Tracer) Close() {}
+
+// WrapReadable wraps an objstorage.Readable with one that generates tracing
+// events.
+func (*Tracer) WrapReadable(
+	ctx context.Context, r objstorage.Readable, fileNum base.DiskFileNum,
+) objstorage.Readable {
+	return r
+}
+
+// WrapWritable wraps an objstorage.Writable with one that generates tracing
+// events.
+func (t *Tracer) WrapWritable(
+	ctx context.Context, w objstorage.Writable, fileNum base.DiskFileNum,
+) objstorage.Writable {
+	return w
+}
+
+// WithReason creates a context that has an associated Reason (which ends up in
+// traces created under that context).
+func WithReason(ctx context.Context, reason Reason) context.Context { return ctx }
+
+// WithBlockType creates a context that has an associated BlockType (which ends up in
+// traces created under that context).
+func WithBlockType(ctx context.Context, blockType BlockType) context.Context { return ctx }
+
+// WithLevel creates a context that has an associated level (which ends up in
+// traces created under that context).
+func WithLevel(ctx context.Context, level int) context.Context { return ctx }

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go

@@ -0,0 +1,410 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build pebble_obj_io_tracing
+// +build pebble_obj_io_tracing
+
+package objiotracing
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+	"time"
+	"unsafe"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// Enabled is used to short circuit tracing-related code in regular builds.
+const Enabled = true
+
+// Tracer manages the writing of object IO traces to files.
+//
+// The tracer runs a background worker goroutine which receives trace event
+// buffers over a channel and dumps them to IOTRACES- files. Wrapper
+// implementations for Readable, ReadHandle, Writable are producers of traces;
+// they maintain internal buffers of events which get flushed to the buffered
+// channel when they get full. This allows for minimal synchronization per IO
+// (as for most of these structures, an instance only allows a single IO at a
+// time).
+type Tracer struct {
+	fs    vfs.FS
+	fsDir string
+
+	handleID atomic.Uint64
+
+	workerStopCh chan struct{}
+	workerDataCh chan eventBuf
+	workerWait   sync.WaitGroup
+}
+
+// Open creates a Tracer which generates trace files in the given directory.
+// Each trace file contains a series of Events (as they are in memory).
+func Open(fs vfs.FS, fsDir string) *Tracer {
+	t := &Tracer{
+		fs:           fs,
+		fsDir:        fsDir,
+		workerStopCh: make(chan struct{}),
+		workerDataCh: make(chan eventBuf, channelBufSize),
+	}
+
+	t.handleID.Store(uint64(rand.NewSource(time.Now().UnixNano()).Int63()))
+
+	t.workerWait.Add(1)
+	go t.workerLoop()
+	return t
+}
+
+// Close the tracer, flushing any remaining events.
+func (t *Tracer) Close() {
+	if t.workerStopCh == nil {
+		return
+	}
+	// Tell the worker to stop and wait for it to finish up.
+	close(t.workerStopCh)
+	t.workerWait.Wait()
+	t.workerStopCh = nil
+}
+
+// WrapWritable wraps an objstorage.Writable with one that generates tracing
+// events.
+func (t *Tracer) WrapWritable(
+	ctx context.Context, w objstorage.Writable, fileNum base.FileNum,
+) objstorage.Writable {
+	return &writable{
+		w:       w,
+		fileNum: fileNum,
+		g:       makeEventGenerator(ctx, t),
+	}
+}
+
+type writable struct {
+	w         objstorage.Writable
+	fileNum   base.FileNum
+	curOffset int64
+	g         eventGenerator
+}
+
+var _ objstorage.Writable = (*writable)(nil)
+
+// Write is part of the objstorage.Writable interface.
+func (w *writable) Write(p []byte) error {
+	w.g.add(context.Background(), Event{
+		Op:      WriteOp,
+		FileNum: w.fileNum,
+		Offset:  w.curOffset,
+		Size:    int64(len(p)),
+	})
+	// If w.w.Write(p) returns an error, a new writable
+	// will be used, so even tho all of p may not have
+	// been written to the underlying "file", it is okay
+	// to add len(p) to curOffset.
+	w.curOffset += int64(len(p))
+	return w.w.Write(p)
+}
+
+// Finish is part of the objstorage.Writable interface.
+func (w *writable) Finish() error {
+	w.g.flush()
+	return w.w.Finish()
+}
+
+// Abort is part of the objstorage.Writable interface.
+func (w *writable) Abort() {
+	w.g.flush()
+	w.w.Abort()
+}
+
+// WrapReadable wraps an objstorage.Readable with one that generates tracing
+// events.
+func (t *Tracer) WrapReadable(
+	ctx context.Context, r objstorage.Readable, fileNum base.FileNum,
+) objstorage.Readable {
+	res := &readable{
+		r:       r,
+		fileNum: fileNum,
+	}
+	res.mu.g = makeEventGenerator(ctx, t)
+	return res
+}
+
+type readable struct {
+	r       objstorage.Readable
+	fileNum base.FileNum
+	mu      struct {
+		sync.Mutex
+		g eventGenerator
+	}
+}
+
+var _ objstorage.Readable = (*readable)(nil)
+
+// ReadAt is part of the objstorage.Readable interface.
+func (r *readable) ReadAt(ctx context.Context, v []byte, off int64) (n int, err error) {
+	r.mu.Lock()
+	r.mu.g.add(ctx, Event{
+		Op:      ReadOp,
+		FileNum: r.fileNum,
+		Offset:  off,
+		Size:    int64(len(v)),
+	})
+	r.mu.Unlock()
+	return r.r.ReadAt(ctx, v, off)
+}
+
+// Close is part of the objstorage.Readable interface.
+func (r *readable) Close() error {
+	r.mu.g.flush()
+	return r.r.Close()
+}
+
+// Size is part of the objstorage.Readable interface.
+func (r *readable) Size() int64 {
+	return r.r.Size()
+}
+
+// NewReadHandle is part of the objstorage.Readable interface.
+func (r *readable) NewReadHandle(ctx context.Context) objstorage.ReadHandle {
+	// It's safe to get the tracer from the generator without the mutex since it never changes.
+	t := r.mu.g.t
+	return &readHandle{
+		rh:       r.r.NewReadHandle(ctx),
+		fileNum:  r.fileNum,
+		handleID: t.handleID.Add(1),
+		g:        makeEventGenerator(ctx, t),
+	}
+}
+
+type readHandle struct {
+	rh       objstorage.ReadHandle
+	fileNum  base.FileNum
+	handleID uint64
+	g        eventGenerator
+}
+
+var _ objstorage.ReadHandle = (*readHandle)(nil)
+
+// ReadAt is part of the objstorage.ReadHandle interface.
+func (rh *readHandle) ReadAt(ctx context.Context, p []byte, off int64) (n int, err error) {
+	rh.g.add(ctx, Event{
+		Op:       ReadOp,
+		FileNum:  rh.fileNum,
+		HandleID: rh.handleID,
+		Offset:   off,
+		Size:     int64(len(p)),
+	})
+	return rh.rh.ReadAt(ctx, p, off)
+}
+
+// Close is part of the objstorage.ReadHandle interface.
+func (rh *readHandle) Close() error {
+	rh.g.flush()
+	return rh.rh.Close()
+}
+
+// MaxReadahead is part of the objstorage.ReadHandle interface.
+func (rh *readHandle) MaxReadahead() {
+	rh.g.add(context.Background(), Event{
+		Op:       MaxReadaheadOp,
+		FileNum:  rh.fileNum,
+		HandleID: rh.handleID,
+	})
+	rh.rh.MaxReadahead()
+}
+
+// RecordCacheHit is part of the objstorage.ReadHandle interface.
+func (rh *readHandle) RecordCacheHit(ctx context.Context, offset, size int64) {
+	rh.g.add(ctx, Event{
+		Op:       RecordCacheHitOp,
+		FileNum:  rh.fileNum,
+		HandleID: rh.handleID,
+		Offset:   offset,
+		Size:     size,
+	})
+	rh.rh.RecordCacheHit(ctx, offset, size)
+}
+
+type ctxInfo struct {
+	reason       Reason
+	blockType    BlockType
+	levelPlusOne uint8
+}
+
+func mergeCtxInfo(base, other ctxInfo) ctxInfo {
+	res := other
+	if res.reason == 0 {
+		res.reason = base.reason
+	}
+	if res.blockType == 0 {
+		res.blockType = base.blockType
+	}
+	if res.levelPlusOne == 0 {
+		res.levelPlusOne = base.levelPlusOne
+	}
+	return res
+}
+
+type ctxInfoKey struct{}
+
+func withInfo(ctx context.Context, info ctxInfo) context.Context {
+	return context.WithValue(ctx, ctxInfoKey{}, info)
+}
+
+func infoFromCtx(ctx context.Context) ctxInfo {
+	res := ctx.Value(ctxInfoKey{})
+	if res == nil {
+		return ctxInfo{}
+	}
+	return res.(ctxInfo)
+}
+
+// WithReason creates a context that has an associated Reason (which ends up in
+// traces created under that context).
+func WithReason(ctx context.Context, reason Reason) context.Context {
+	info := infoFromCtx(ctx)
+	info.reason = reason
+	return withInfo(ctx, info)
+}
+
+// WithBlockType creates a context that has an associated BlockType (which ends up in
+// traces created under that context).
+func WithBlockType(ctx context.Context, blockType BlockType) context.Context {
+	info := infoFromCtx(ctx)
+	info.blockType = blockType
+	return withInfo(ctx, info)
+}
+
+// WithLevel creates a context that has an associated level (which ends up in
+// traces created under that context).
+func WithLevel(ctx context.Context, level int) context.Context {
+	info := infoFromCtx(ctx)
+	info.levelPlusOne = uint8(level) + 1
+	return withInfo(ctx, info)
+}
+
+const (
+	eventSize            = int(unsafe.Sizeof(Event{}))
+	targetEntriesPerFile = 256 * 1024 * 1024 / eventSize // 256MB files
+	eventsPerBuf         = 16
+	channelBufSize       = 512 * 1024 / eventsPerBuf // 512K events.
+	bytesPerFileSync     = 128 * 1024
+)
+
+type eventBuf struct {
+	events [eventsPerBuf]Event
+	num    int
+}
+
+type eventGenerator struct {
+	t           *Tracer
+	baseCtxInfo ctxInfo
+	buf         eventBuf
+}
+
+func makeEventGenerator(ctx context.Context, t *Tracer) eventGenerator {
+	return eventGenerator{
+		t:           t,
+		baseCtxInfo: infoFromCtx(ctx),
+	}
+}
+
+func (g *eventGenerator) flush() {
+	if g.buf.num > 0 {
+		g.t.workerDataCh <- g.buf
+		g.buf.num = 0
+	}
+}
+
+func (g *eventGenerator) add(ctx context.Context, e Event) {
+	e.StartUnixNano = time.Now().UnixNano()
+	info := infoFromCtx(ctx)
+	info = mergeCtxInfo(g.baseCtxInfo, info)
+	e.Reason = info.reason
+	e.BlockType = info.blockType
+	e.LevelPlusOne = info.levelPlusOne
+	if g.buf.num == eventsPerBuf {
+		g.flush()
+	}
+	g.buf.events[g.buf.num] = e
+	g.buf.num++
+}
+
+type workerState struct {
+	curFile          vfs.File
+	curBW            *bufio.Writer
+	numEntriesInFile int
+}
+
+func (t *Tracer) workerLoop() {
+	defer t.workerWait.Done()
+	stopCh := t.workerStopCh
+	dataCh := t.workerDataCh
+	var state workerState
+	t.workerNewFile(&state)
+	for {
+		select {
+		case <-stopCh:
+			close(dataCh)
+			// Flush any remaining traces.
+			for data := range dataCh {
+				t.workerWriteTraces(&state, data)
+			}
+			t.workerCloseFile(&state)
+			return
+
+		case data := <-dataCh:
+			t.workerWriteTraces(&state, data)
+		}
+	}
+}
+
+func (t *Tracer) workerWriteTraces(state *workerState, data eventBuf) {
+	if state.numEntriesInFile >= targetEntriesPerFile {
+		t.workerCloseFile(state)
+		t.workerNewFile(state)
+	}
+	state.numEntriesInFile += data.num
+	p := unsafe.Pointer(&data.events[0])
+	b := unsafe.Slice((*byte)(p), eventSize*data.num)
+	if _, err := state.curBW.Write(b); err != nil {
+		panic(err)
+	}
+}
+
+func (t *Tracer) workerNewFile(state *workerState) {
+	filename := fmt.Sprintf("IOTRACES-%s", time.Now().UTC().Format(time.RFC3339Nano))
+
+	file, err := t.fs.Create(t.fs.PathJoin(t.fsDir, filename))
+	if err != nil {
+		panic(err)
+	}
+	file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{
+		BytesPerSync: bytesPerFileSync,
+	})
+	state.curFile = file
+	state.curBW = bufio.NewWriter(file)
+	state.numEntriesInFile = 0
+}
+
+func (t *Tracer) workerCloseFile(state *workerState) {
+	if state.curFile != nil {
+		if err := state.curBW.Flush(); err != nil {
+			panic(err)
+		}
+		if err := state.curFile.Sync(); err != nil {
+			panic(err)
+		}
+		if err := state.curFile.Close(); err != nil {
+			panic(err)
+		}
+		state.curFile = nil
+		state.curBW = nil
+	}
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go

@@ -0,0 +1,460 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"io"
+	"os"
+	"sort"
+	"sync"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/errors/oserror"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
+	"github.com/cockroachdb/pebble/objstorage/shared"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// provider is the implementation of objstorage.Provider.
+type provider struct {
+	st Settings
+
+	fsDir vfs.File
+
+	tracer *objiotracing.Tracer
+
+	shared sharedSubsystem
+
+	mu struct {
+		sync.RWMutex
+
+		shared struct {
+			// catalogBatch accumulates shared object creations and deletions until
+			// Sync is called.
+			catalogBatch sharedobjcat.Batch
+		}
+
+		// localObjectsChanged is set if non-shared objects were created or deleted
+		// but Sync was not yet called.
+		localObjectsChanged bool
+
+		// knownObjects maintains information about objects that are known to the provider.
+		// It is initialized with the list of files in the manifest when we open a DB.
+		knownObjects map[base.DiskFileNum]objstorage.ObjectMetadata
+
+		// protectedObjects are objects that cannot be unreferenced because they
+		// have outstanding SharedObjectBackingHandles. The value is a count of outstanding handles
+		protectedObjects map[base.DiskFileNum]int
+	}
+}
+
+var _ objstorage.Provider = (*provider)(nil)
+
+// Settings that must be specified when creating the provider.
+type Settings struct {
+	Logger base.Logger
+
+	// Local filesystem configuration.
+	FS        vfs.FS
+	FSDirName string
+
+	// FSDirInitialListing is a listing of FSDirName at the time of calling Open.
+	//
+	// This is an optional optimization to avoid double listing on Open when the
+	// higher layer already has a listing. When nil, we obtain the listing on
+	// Open.
+	FSDirInitialListing []string
+
+	// Cleaner cleans obsolete files from the local filesystem.
+	//
+	// The default cleaner uses the DeleteCleaner.
+	FSCleaner base.Cleaner
+
+	// NoSyncOnClose decides whether the implementation will enforce a
+	// close-time synchronization (e.g., fdatasync() or sync_file_range())
+	// on files it writes to. Setting this to true removes the guarantee for a
+	// sync on close. Some implementations can still issue a non-blocking sync.
+	NoSyncOnClose bool
+
+	// BytesPerSync enables periodic syncing of files in order to smooth out
+	// writes to disk. This option does not provide any persistence guarantee, but
+	// is used to avoid latency spikes if the OS automatically decides to write
+	// out a large chunk of dirty filesystem buffers.
+	BytesPerSync int
+
+	// Fields here are set only if the provider is to support shared objects
+	// (experimental).
+	Shared struct {
+		Storage shared.Storage
+	}
+}
+
+// DefaultSettings initializes default settings (with no shared storage),
+// suitable for tests and tools.
+func DefaultSettings(fs vfs.FS, dirName string) Settings {
+	return Settings{
+		Logger:        base.DefaultLogger,
+		FS:            fs,
+		FSDirName:     dirName,
+		FSCleaner:     base.DeleteCleaner{},
+		NoSyncOnClose: false,
+		BytesPerSync:  512 * 1024, // 512KB
+	}
+}
+
+// Open creates the provider.
+func Open(settings Settings) (objstorage.Provider, error) {
+	return open(settings)
+}
+
+func open(settings Settings) (p *provider, _ error) {
+	fsDir, err := settings.FS.OpenDir(settings.FSDirName)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if p == nil {
+			fsDir.Close()
+		}
+	}()
+
+	p = &provider{
+		st:    settings,
+		fsDir: fsDir,
+	}
+	p.mu.knownObjects = make(map[base.DiskFileNum]objstorage.ObjectMetadata)
+	p.mu.protectedObjects = make(map[base.DiskFileNum]int)
+
+	if objiotracing.Enabled {
+		p.tracer = objiotracing.Open(settings.FS, settings.FSDirName)
+	}
+
+	// Add local FS objects.
+	if err := p.vfsInit(); err != nil {
+		return nil, err
+	}
+
+	// Initialize shared subsystem (if configured) and add shared objects.
+	if err := p.sharedInit(); err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}
+
+// Close is part of the objstorage.Provider interface.
+func (p *provider) Close() error {
+	var err error
+	if p.fsDir != nil {
+		err = p.fsDir.Close()
+		p.fsDir = nil
+	}
+	if objiotracing.Enabled {
+		if p.tracer != nil {
+			p.tracer.Close()
+			p.tracer = nil
+		}
+	}
+	return err
+}
+
+// OpenForReading opens an existing object.
+func (p *provider) OpenForReading(
+	ctx context.Context,
+	fileType base.FileType,
+	fileNum base.DiskFileNum,
+	opts objstorage.OpenOptions,
+) (objstorage.Readable, error) {
+	meta, err := p.Lookup(fileType, fileNum)
+	if err != nil {
+		if opts.MustExist {
+			p.st.Logger.Fatalf("%v", err)
+		}
+		return nil, err
+	}
+
+	var r objstorage.Readable
+	if !meta.IsShared() {
+		r, err = p.vfsOpenForReading(ctx, fileType, fileNum, opts)
+	} else {
+		r, err = p.sharedOpenForReading(ctx, meta, opts)
+	}
+	if err != nil {
+		return nil, err
+	}
+	if objiotracing.Enabled {
+		r = p.tracer.WrapReadable(ctx, r, fileNum)
+	}
+	return r, nil
+}
+
+// Create creates a new object and opens it for writing.
+//
+// The object is not guaranteed to be durable (accessible in case of crashes)
+// until Sync is called.
+func (p *provider) Create(
+	ctx context.Context,
+	fileType base.FileType,
+	fileNum base.DiskFileNum,
+	opts objstorage.CreateOptions,
+) (w objstorage.Writable, meta objstorage.ObjectMetadata, err error) {
+	if opts.PreferSharedStorage && p.st.Shared.Storage != nil {
+		w, meta, err = p.sharedCreate(ctx, fileType, fileNum, opts)
+	} else {
+		w, meta, err = p.vfsCreate(ctx, fileType, fileNum)
+	}
+	if err != nil {
+		err = errors.Wrapf(err, "creating object %s", errors.Safe(fileNum))
+		return nil, objstorage.ObjectMetadata{}, err
+	}
+	p.addMetadata(meta)
+	if objiotracing.Enabled {
+		w = p.tracer.WrapWritable(ctx, w, fileNum)
+	}
+	return w, meta, nil
+}
+
+// Remove removes an object.
+//
+// Note that if the object is shared, the object is only (conceptually) removed
+// from this provider. If other providers have references on the shared object,
+// it will not be removed.
+//
+// The object is not guaranteed to be durably removed until Sync is called.
+func (p *provider) Remove(fileType base.FileType, fileNum base.DiskFileNum) error {
+	meta, err := p.Lookup(fileType, fileNum)
+	if err != nil {
+		return err
+	}
+
+	if !meta.IsShared() {
+		err = p.vfsRemove(fileType, fileNum)
+	} else {
+		// TODO(radu): implement shared object removal (i.e. deref).
+		err = p.sharedUnref(meta)
+	}
+	if err != nil && !p.IsNotExistError(err) {
+		// We want to be able to retry a Remove, so we keep the object in our list.
+		// TODO(radu): we should mark the object as "zombie" and not allow any other
+		// operations.
+		return errors.Wrapf(err, "removing object %s", errors.Safe(fileNum))
+	}
+
+	p.removeMetadata(fileNum)
+	return err
+}
+
+// IsNotExistError is part of the objstorage.Provider interface.
+func (p *provider) IsNotExistError(err error) bool {
+	if oserror.IsNotExist(err) {
+		return true
+	}
+	if p.sharedStorage() != nil && p.sharedStorage().IsNotExistError(err) {
+		return true
+	}
+	return false
+}
+
+// Sync flushes the metadata from creation or removal of objects since the last Sync.
+func (p *provider) Sync() error {
+	if err := p.vfsSync(); err != nil {
+		return err
+	}
+	if err := p.sharedSync(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// LinkOrCopyFromLocal creates a new object that is either a copy of a given
+// local file or a hard link (if the new object is created on the same FS, and
+// if the FS supports it).
+//
+// The object is not guaranteed to be durable (accessible in case of crashes)
+// until Sync is called.
+func (p *provider) LinkOrCopyFromLocal(
+	ctx context.Context,
+	srcFS vfs.FS,
+	srcFilePath string,
+	dstFileType base.FileType,
+	dstFileNum base.DiskFileNum,
+	opts objstorage.CreateOptions,
+) (objstorage.ObjectMetadata, error) {
+	shared := opts.PreferSharedStorage && p.st.Shared.Storage != nil
+	if !shared && srcFS == p.st.FS {
+		// Wrap the normal filesystem with one which wraps newly created files with
+		// vfs.NewSyncingFile.
+		fs := vfs.NewSyncingFS(p.st.FS, vfs.SyncingFileOptions{
+			NoSyncOnClose: p.st.NoSyncOnClose,
+			BytesPerSync:  p.st.BytesPerSync,
+		})
+		dstPath := p.vfsPath(dstFileType, dstFileNum)
+		if err := vfs.LinkOrCopy(fs, srcFilePath, dstPath); err != nil {
+			return objstorage.ObjectMetadata{}, err
+		}
+
+		meta := objstorage.ObjectMetadata{
+			DiskFileNum: dstFileNum,
+			FileType:    dstFileType,
+		}
+		p.addMetadata(meta)
+		return meta, nil
+	}
+	// Create the object and copy the data.
+	w, meta, err := p.Create(ctx, dstFileType, dstFileNum, opts)
+	if err != nil {
+		return objstorage.ObjectMetadata{}, err
+	}
+	f, err := srcFS.Open(srcFilePath, vfs.SequentialReadsOption)
+	if err != nil {
+		return objstorage.ObjectMetadata{}, err
+	}
+	defer f.Close()
+	buf := make([]byte, 64*1024)
+	for {
+		n, readErr := f.Read(buf)
+		if readErr != nil && readErr != io.EOF {
+			w.Abort()
+			return objstorage.ObjectMetadata{}, readErr
+		}
+
+		if n > 0 {
+			if err := w.Write(buf[:n]); err != nil {
+				w.Abort()
+				return objstorage.ObjectMetadata{}, err
+			}
+		}
+
+		if readErr == io.EOF {
+			break
+		}
+	}
+	if err := w.Finish(); err != nil {
+		return objstorage.ObjectMetadata{}, err
+	}
+	return meta, nil
+}
+
+// Lookup is part of the objstorage.Provider interface.
+func (p *provider) Lookup(
+	fileType base.FileType, fileNum base.DiskFileNum,
+) (objstorage.ObjectMetadata, error) {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	meta, ok := p.mu.knownObjects[fileNum]
+	if !ok {
+		return objstorage.ObjectMetadata{}, errors.Wrapf(
+			os.ErrNotExist,
+			"file %s (type %d) unknown to the objstorage provider",
+			errors.Safe(fileNum), errors.Safe(fileType),
+		)
+	}
+	if meta.FileType != fileType {
+		return objstorage.ObjectMetadata{}, errors.AssertionFailedf(
+			"file %s type mismatch (known type %d, expected type %d)",
+			errors.Safe(fileNum), errors.Safe(meta.FileType), errors.Safe(fileType),
+		)
+	}
+	return meta, nil
+}
+
+// Path is part of the objstorage.Provider interface.
+func (p *provider) Path(meta objstorage.ObjectMetadata) string {
+	if !meta.IsShared() {
+		return p.vfsPath(meta.FileType, meta.DiskFileNum)
+	}
+	return p.sharedPath(meta)
+}
+
+// Size returns the size of the object.
+func (p *provider) Size(meta objstorage.ObjectMetadata) (int64, error) {
+	if !meta.IsShared() {
+		return p.vfsSize(meta.FileType, meta.DiskFileNum)
+	}
+	return p.sharedSize(meta)
+}
+
+// List is part of the objstorage.Provider interface.
+func (p *provider) List() []objstorage.ObjectMetadata {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	res := make([]objstorage.ObjectMetadata, 0, len(p.mu.knownObjects))
+	for _, meta := range p.mu.knownObjects {
+		res = append(res, meta)
+	}
+	sort.Slice(res, func(i, j int) bool {
+		return res[i].DiskFileNum.FileNum() < res[j].DiskFileNum.FileNum()
+	})
+	return res
+}
+
+func (p *provider) addMetadata(meta objstorage.ObjectMetadata) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.mu.knownObjects[meta.DiskFileNum] = meta
+	if meta.IsShared() {
+		p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
+			FileNum:        meta.DiskFileNum,
+			FileType:       meta.FileType,
+			CreatorID:      meta.Shared.CreatorID,
+			CreatorFileNum: meta.Shared.CreatorFileNum,
+			CleanupMethod:  meta.Shared.CleanupMethod,
+		})
+	} else {
+		p.mu.localObjectsChanged = true
+	}
+}
+
+func (p *provider) removeMetadata(fileNum base.DiskFileNum) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	meta, ok := p.mu.knownObjects[fileNum]
+	if !ok {
+		return
+	}
+	delete(p.mu.knownObjects, fileNum)
+	if meta.IsShared() {
+		p.mu.shared.catalogBatch.DeleteObject(fileNum)
+	} else {
+		p.mu.localObjectsChanged = true
+	}
+}
+
+// protectObject prevents the unreferencing of a shared object until
+// unprotectObject is called.
+func (p *provider) protectObject(fileNum base.DiskFileNum) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.mu.protectedObjects[fileNum] = p.mu.protectedObjects[fileNum] + 1
+}
+
+func (p *provider) unprotectObject(fileNum base.DiskFileNum) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	v := p.mu.protectedObjects[fileNum]
+	if invariants.Enabled && v == 0 {
+		panic("invalid protection count")
+	}
+	if v > 1 {
+		p.mu.protectedObjects[fileNum] = v - 1
+	} else {
+		delete(p.mu.protectedObjects, fileNum)
+		// TODO(radu): check if the object is still in knownObject; if not, unref it
+		// now.
+	}
+}
+
+func (p *provider) isProtected(fileNum base.DiskFileNum) bool {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.mu.protectedObjects[fileNum] > 0
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go

@@ -0,0 +1,200 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+const (
+	// Constants for dynamic readahead of data blocks. Note that the size values
+	// make sense as some multiple of the default block size; and they should
+	// both be larger than the default block size.
+	minFileReadsForReadahead = 2
+	// TODO(bilal): Have the initial size value be a factor of the block size,
+	// as opposed to a hardcoded value.
+	initialReadaheadSize = 64 << 10  /* 64KB */
+	maxReadaheadSize     = 256 << 10 /* 256KB */
+)
+
+// readaheadState contains state variables related to readahead. Updated on
+// file reads.
+type readaheadState struct {
+	// Number of sequential reads.
+	numReads int64
+	// Size issued to the next call to Prefetch. Starts at or above
+	// initialReadaheadSize and grows exponentially until maxReadaheadSize.
+	size int64
+	// prevSize is the size used in the last Prefetch call.
+	prevSize int64
+	// The byte offset up to which the OS has been asked to read ahead / cached.
+	// When reading ahead, reads up to this limit should not incur an IO
+	// operation. Reads after this limit can benefit from a new call to
+	// Prefetch.
+	limit int64
+}
+
+func makeReadaheadState() readaheadState {
+	return readaheadState{
+		size: initialReadaheadSize,
+	}
+}
+
+func (rs *readaheadState) recordCacheHit(offset, blockLength int64) {
+	currentReadEnd := offset + blockLength
+	if rs.numReads >= minFileReadsForReadahead {
+		if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
+			// This is a read that would have resulted in a readahead, had it
+			// not been a cache hit.
+			rs.limit = currentReadEnd
+			return
+		}
+		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize {
+			// We read too far away from rs.limit to benefit from readahead in
+			// any scenario. Reset all variables.
+			rs.numReads = 1
+			rs.limit = currentReadEnd
+			rs.size = initialReadaheadSize
+			rs.prevSize = 0
+			return
+		}
+		// Reads in the range [rs.limit - rs.prevSize, rs.limit] end up
+		// here. This is a read that is potentially benefitting from a past
+		// readahead.
+		return
+	}
+	if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
+		// Blocks are being read sequentially and would benefit from readahead
+		// down the line.
+		rs.numReads++
+		return
+	}
+	// We read too far ahead of the last read, or before it. This indicates
+	// a random read, where readahead is not desirable. Reset all variables.
+	rs.numReads = 1
+	rs.limit = currentReadEnd
+	rs.size = initialReadaheadSize
+	rs.prevSize = 0
+}
+
+// maybeReadahead updates state and determines whether to issue a readahead /
+// prefetch call for a block read at offset for blockLength bytes.
+// Returns a size value (greater than 0) that should be prefetched if readahead
+// would be beneficial.
+func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
+	currentReadEnd := offset + blockLength
+	if rs.numReads >= minFileReadsForReadahead {
+		// The minimum threshold of sequential reads to justify reading ahead
+		// has been reached.
+		// There are two intervals: the interval being read:
+		// [offset, currentReadEnd]
+		// as well as the interval where a read would benefit from read ahead:
+		// [rs.limit, rs.limit + rs.size]
+		// We increase the latter interval to
+		// [rs.limit, rs.limit + maxReadaheadSize] to account for cases where
+		// readahead may not be beneficial with a small readahead size, but over
+		// time the readahead size would increase exponentially to make it
+		// beneficial.
+		if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
+			// We are doing a read in the interval ahead of
+			// the last readahead range. In the diagrams below, ++++ is the last
+			// readahead range, ==== is the range represented by
+			// [rs.limit, rs.limit + maxReadaheadSize], and ---- is the range
+			// being read.
+			//
+			//               rs.limit           rs.limit + maxReadaheadSize
+			//         ++++++++++|===========================|
+			//
+			//              |-------------|
+			//            offset       currentReadEnd
+			//
+			// This case is also possible, as are all cases with an overlap
+			// between [rs.limit, rs.limit + maxReadaheadSize] and [offset,
+			// currentReadEnd]:
+			//
+			//               rs.limit           rs.limit + maxReadaheadSize
+			//         ++++++++++|===========================|
+			//
+			//                                            |-------------|
+			//                                         offset       currentReadEnd
+			//
+			//
+			rs.numReads++
+			rs.limit = offset + rs.size
+			rs.prevSize = rs.size
+			// Increase rs.size for the next read.
+			rs.size *= 2
+			if rs.size > maxReadaheadSize {
+				rs.size = maxReadaheadSize
+			}
+			return rs.prevSize
+		}
+		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize {
+			// The above conditional has rs.limit > rs.prevSize to confirm that
+			// rs.limit - rs.prevSize would not underflow.
+			// We read too far away from rs.limit to benefit from readahead in
+			// any scenario. Reset all variables.
+			// The case where we read too far ahead:
+			//
+			// (rs.limit - rs.prevSize)    (rs.limit)   (rs.limit + maxReadaheadSize)
+			//                    |+++++++++++++|=============|
+			//
+			//                                                  |-------------|
+			//                                             offset       currentReadEnd
+			//
+			// Or too far behind:
+			//
+			// (rs.limit - rs.prevSize)    (rs.limit)   (rs.limit + maxReadaheadSize)
+			//                    |+++++++++++++|=============|
+			//
+			//    |-------------|
+			// offset       currentReadEnd
+			//
+			rs.numReads = 1
+			rs.limit = currentReadEnd
+			rs.size = initialReadaheadSize
+			rs.prevSize = 0
+
+			return 0
+		}
+		// Reads in the range [rs.limit - rs.prevSize, rs.limit] end up
+		// here. This is a read that is potentially benefitting from a past
+		// readahead, but there's no reason to issue a readahead call at the
+		// moment.
+		//
+		// (rs.limit - rs.prevSize)            (rs.limit + maxReadaheadSize)
+		//                    |+++++++++++++|===============|
+		//                             (rs.limit)
+		//
+		//                        |-------|
+		//                     offset    currentReadEnd
+		//
+		rs.numReads++
+		return 0
+	}
+	if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
+		// Blocks are being read sequentially and would benefit from readahead
+		// down the line.
+		//
+		//                       (rs.limit)   (rs.limit + maxReadaheadSize)
+		//                         |=============|
+		//
+		//                    |-------|
+		//                offset    currentReadEnd
+		//
+		rs.numReads++
+		return 0
+	}
+	// We read too far ahead of the last read, or before it. This indicates
+	// a random read, where readahead is not desirable. Reset all variables.
+	//
+	// (rs.limit - maxReadaheadSize)  (rs.limit)   (rs.limit + maxReadaheadSize)
+	//                     |+++++++++++++|=============|
+	//
+	//                                                    |-------|
+	//                                                offset    currentReadEnd
+	//
+	rs.numReads = 1
+	rs.limit = currentReadEnd
+	rs.size = initialReadaheadSize
+	rs.prevSize = 0
+	return 0
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared.go

@@ -0,0 +1,255 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
+	"github.com/cockroachdb/pebble/objstorage/shared"
+)
+
+// sharedSubsystem contains the provider fields related to shared storage.
+// All fields remain unset if shared storage is not configured.
+type sharedSubsystem struct {
+	catalog *sharedobjcat.Catalog
+
+	// checkRefsOnOpen controls whether we check the ref marker file when opening
+	// an object. Normally this is true when invariants are enabled (but the provider
+	// test tweaks this field).
+	checkRefsOnOpen bool
+
+	// initialized guards access to the creatorID field.
+	initialized atomic.Bool
+	creatorID   objstorage.CreatorID
+	initOnce    sync.Once
+}
+
+func (ss *sharedSubsystem) init(creatorID objstorage.CreatorID) {
+	ss.initOnce.Do(func() {
+		ss.creatorID = creatorID
+		ss.initialized.Store(true)
+	})
+}
+
+func (p *provider) sharedStorage() shared.Storage {
+	return p.st.Shared.Storage
+}
+
+// sharedInit initializes the shared object subsystem (if configured) and finds
+// any shared objects.
+func (p *provider) sharedInit() error {
+	if p.sharedStorage() == nil {
+		return nil
+	}
+	catalog, contents, err := sharedobjcat.Open(p.st.FS, p.st.FSDirName)
+	if err != nil {
+		return errors.Wrapf(err, "pebble: could not open shared object catalog")
+	}
+	p.shared.catalog = catalog
+	p.shared.checkRefsOnOpen = invariants.Enabled
+
+	// The creator ID may or may not be initialized yet.
+	if contents.CreatorID.IsSet() {
+		p.shared.init(contents.CreatorID)
+		p.st.Logger.Infof("shared storage configured; creatorID = %s", contents.CreatorID)
+	} else {
+		p.st.Logger.Infof("shared storage configured; no creatorID yet")
+	}
+
+	for _, meta := range contents.Objects {
+		o := objstorage.ObjectMetadata{
+			DiskFileNum: meta.FileNum,
+			FileType:    meta.FileType,
+		}
+		o.Shared.CreatorID = meta.CreatorID
+		o.Shared.CreatorFileNum = meta.CreatorFileNum
+		o.Shared.CleanupMethod = meta.CleanupMethod
+		p.mu.knownObjects[o.DiskFileNum] = o
+	}
+	return nil
+}
+
+// SetCreatorID is part of the objstorage.Provider interface.
+func (p *provider) SetCreatorID(creatorID objstorage.CreatorID) error {
+	if p.sharedStorage() == nil {
+		return errors.AssertionFailedf("attempt to set CreatorID but shared storage not enabled")
+	}
+	// Note: this call is a cheap no-op if the creator ID was already set. This
+	// call also checks if we are trying to change the ID.
+	if err := p.shared.catalog.SetCreatorID(creatorID); err != nil {
+		return err
+	}
+	if !p.shared.initialized.Load() {
+		p.st.Logger.Infof("shared storage creatorID set to %s", creatorID)
+		p.shared.init(creatorID)
+	}
+	return nil
+}
+
+func (p *provider) sharedCheckInitialized() error {
+	if p.sharedStorage() == nil {
+		return errors.Errorf("shared object support not configured")
+	}
+	if !p.shared.initialized.Load() {
+		return errors.Errorf("shared object support not available: shared creator ID not yet set")
+	}
+	return nil
+}
+
+func (p *provider) sharedSync() error {
+	batch := func() sharedobjcat.Batch {
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		res := p.mu.shared.catalogBatch.Copy()
+		p.mu.shared.catalogBatch.Reset()
+		return res
+	}()
+
+	if batch.IsEmpty() {
+		return nil
+	}
+
+	err := p.shared.catalog.ApplyBatch(batch)
+	if err != nil {
+		// We have to put back the batch (for the next Sync).
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		batch.Append(p.mu.shared.catalogBatch)
+		p.mu.shared.catalogBatch = batch
+		return err
+	}
+
+	return nil
+}
+
+func (p *provider) sharedPath(meta objstorage.ObjectMetadata) string {
+	return "shared://" + sharedObjectName(meta)
+}
+
+// sharedCreateRef creates a reference marker object.
+func (p *provider) sharedCreateRef(meta objstorage.ObjectMetadata) error {
+	if meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
+		return nil
+	}
+	refName := p.sharedObjectRefName(meta)
+	writer, err := p.sharedStorage().CreateObject(refName)
+	if err == nil {
+		// The object is empty, just close the writer.
+		err = writer.Close()
+	}
+	if err != nil {
+		return errors.Wrapf(err, "creating marker object %q", refName)
+	}
+	return nil
+}
+
+func (p *provider) sharedCreate(
+	_ context.Context,
+	fileType base.FileType,
+	fileNum base.DiskFileNum,
+	opts objstorage.CreateOptions,
+) (objstorage.Writable, objstorage.ObjectMetadata, error) {
+	if err := p.sharedCheckInitialized(); err != nil {
+		return nil, objstorage.ObjectMetadata{}, err
+	}
+	meta := objstorage.ObjectMetadata{
+		DiskFileNum: fileNum,
+		FileType:    fileType,
+	}
+	meta.Shared.CreatorID = p.shared.creatorID
+	meta.Shared.CreatorFileNum = fileNum
+	meta.Shared.CleanupMethod = opts.SharedCleanupMethod
+
+	objName := sharedObjectName(meta)
+	writer, err := p.sharedStorage().CreateObject(objName)
+	if err != nil {
+		return nil, objstorage.ObjectMetadata{}, errors.Wrapf(err, "creating object %q", objName)
+	}
+	return &sharedWritable{
+		p:             p,
+		meta:          meta,
+		storageWriter: writer,
+	}, meta, nil
+}
+
+func (p *provider) sharedOpenForReading(
+	ctx context.Context, meta objstorage.ObjectMetadata, opts objstorage.OpenOptions,
+) (objstorage.Readable, error) {
+	if err := p.sharedCheckInitialized(); err != nil {
+		return nil, err
+	}
+	// Verify we have a reference on this object; for performance reasons, we only
+	// do this in testing scenarios.
+	if p.shared.checkRefsOnOpen && meta.Shared.CleanupMethod == objstorage.SharedRefTracking {
+		refName := p.sharedObjectRefName(meta)
+		if _, err := p.sharedStorage().Size(refName); err != nil {
+			if p.sharedStorage().IsNotExistError(err) {
+				if opts.MustExist {
+					p.st.Logger.Fatalf("marker object %q does not exist", refName)
+					// TODO(radu): maybe list references for the object.
+				}
+				return nil, errors.Errorf("marker object %q does not exist", refName)
+			}
+			return nil, errors.Wrapf(err, "checking marker object %q", refName)
+		}
+	}
+	objName := sharedObjectName(meta)
+	reader, size, err := p.sharedStorage().ReadObject(ctx, objName)
+	if err != nil {
+		if opts.MustExist && p.sharedStorage().IsNotExistError(err) {
+			p.st.Logger.Fatalf("object %q does not exist", objName)
+			// TODO(radu): maybe list references for the object.
+		}
+		return nil, err
+	}
+	return newSharedReadable(reader, size), nil
+}
+
+func (p *provider) sharedSize(meta objstorage.ObjectMetadata) (int64, error) {
+	if err := p.sharedCheckInitialized(); err != nil {
+		return 0, err
+	}
+	objName := sharedObjectName(meta)
+	return p.sharedStorage().Size(objName)
+}
+
+// sharedUnref implements object "removal" with the shared backend. The ref
+// marker object is removed and the backing object is removed only if there are
+// no other ref markers.
+func (p *provider) sharedUnref(meta objstorage.ObjectMetadata) error {
+	if meta.Shared.CleanupMethod == objstorage.SharedNoCleanup {
+		// Never delete objects in this mode.
+		return nil
+	}
+	if p.isProtected(meta.DiskFileNum) {
+		// TODO(radu): we need a mechanism to unref the object when it becomes
+		// unprotected.
+		return nil
+	}
+
+	refName := p.sharedObjectRefName(meta)
+	// Tolerate a not-exists error.
+	if err := p.sharedStorage().Delete(refName); err != nil && !p.sharedStorage().IsNotExistError(err) {
+		return err
+	}
+	otherRefs, err := p.sharedStorage().List(sharedObjectRefPrefix(meta), "" /* delimiter */)
+	if err != nil {
+		return err
+	}
+	if len(otherRefs) == 0 {
+		objName := sharedObjectName(meta)
+		if err := p.sharedStorage().Delete(objName); err != nil && !p.sharedStorage().IsNotExistError(err) {
+			return err
+		}
+	}
+	return nil
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_backing.go

@@ -0,0 +1,240 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
+)
+
+const (
+	tagCreatorID      = 1
+	tagCreatorFileNum = 2
+	tagCleanupMethod  = 3
+	// tagRefCheckID encodes the information for a ref marker that needs to be
+	// checked when attaching this object to another provider. This is set to the
+	// creator ID and FileNum for the provider that encodes the backing, and
+	// allows the "target" provider to check that the "source" provider kept its
+	// reference on the object alive.
+	tagRefCheckID = 4
+
+	// Any new tags that don't have the tagNotSafeToIgnoreMask bit set must be
+	// followed by the length of the data (so they can be skipped).
+
+	// Any new tags that have the tagNotSafeToIgnoreMask bit set cause errors if
+	// they are encountered by earlier code that doesn't know the tag.
+	tagNotSafeToIgnoreMask = 64
+)
+
+func (p *provider) encodeSharedObjectBacking(
+	meta *objstorage.ObjectMetadata,
+) (objstorage.SharedObjectBacking, error) {
+	if !meta.IsShared() {
+		return nil, errors.AssertionFailedf("object %s not on shared storage", meta.DiskFileNum)
+	}
+
+	buf := make([]byte, 0, binary.MaxVarintLen64*4)
+	buf = binary.AppendUvarint(buf, tagCreatorID)
+	buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorID))
+	// TODO(radu): encode file type as well?
+	buf = binary.AppendUvarint(buf, tagCreatorFileNum)
+	buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorFileNum.FileNum()))
+	buf = binary.AppendUvarint(buf, tagCleanupMethod)
+	buf = binary.AppendUvarint(buf, uint64(meta.Shared.CleanupMethod))
+	if meta.Shared.CleanupMethod == objstorage.SharedRefTracking {
+		buf = binary.AppendUvarint(buf, tagRefCheckID)
+		buf = binary.AppendUvarint(buf, uint64(p.shared.creatorID))
+		buf = binary.AppendUvarint(buf, uint64(meta.DiskFileNum.FileNum()))
+	}
+	return buf, nil
+}
+
+type sharedObjectBackingHandle struct {
+	backing objstorage.SharedObjectBacking
+	fileNum base.DiskFileNum
+	p       *provider
+}
+
+func (s *sharedObjectBackingHandle) Get() (objstorage.SharedObjectBacking, error) {
+	if s.backing == nil {
+		return nil, errors.Errorf("SharedObjectBackingHandle.Get() called after Close()")
+	}
+	return s.backing, nil
+}
+
+func (s *sharedObjectBackingHandle) Close() {
+	if s.backing != nil {
+		s.backing = nil
+		s.p.unprotectObject(s.fileNum)
+	}
+}
+
+var _ objstorage.SharedObjectBackingHandle = (*sharedObjectBackingHandle)(nil)
+
+// SharedObjectBacking is part of the objstorage.Provider interface.
+func (p *provider) SharedObjectBacking(
+	meta *objstorage.ObjectMetadata,
+) (objstorage.SharedObjectBackingHandle, error) {
+	backing, err := p.encodeSharedObjectBacking(meta)
+	if err != nil {
+		return nil, err
+	}
+	p.protectObject(meta.DiskFileNum)
+	return &sharedObjectBackingHandle{
+		backing: backing,
+		fileNum: meta.DiskFileNum,
+		p:       p,
+	}, nil
+}
+
+type decodedBacking struct {
+	meta objstorage.ObjectMetadata
+	// refToCheck is set only when meta.Shared.CleanupMethod is RefTracking
+	refToCheck struct {
+		creatorID objstorage.CreatorID
+		fileNum   base.DiskFileNum
+	}
+}
+
+// decodeSharedObjectBacking decodes the shared object metadata.
+// Returns the object metadata and (optionally) the creator ID of the provider
+// that encoded the backing whose ref marker needs to be checked.
+func decodeSharedObjectBacking(
+	fileType base.FileType, fileNum base.DiskFileNum, buf objstorage.SharedObjectBacking,
+) (decodedBacking, error) {
+	var creatorID, creatorFileNum, cleanupMethod, refCheckCreatorID, refCheckFileNum uint64
+	br := bytes.NewReader(buf)
+	for {
+		tag, err := binary.ReadUvarint(br)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return decodedBacking{}, err
+		}
+		switch tag {
+		case tagCreatorID:
+			creatorID, err = binary.ReadUvarint(br)
+
+		case tagCreatorFileNum:
+			creatorFileNum, err = binary.ReadUvarint(br)
+
+		case tagCleanupMethod:
+			cleanupMethod, err = binary.ReadUvarint(br)
+
+		case tagRefCheckID:
+			refCheckCreatorID, err = binary.ReadUvarint(br)
+			if err == nil {
+				refCheckFileNum, err = binary.ReadUvarint(br)
+			}
+
+		default:
+			// Ignore unknown tags, unless they're not safe to ignore.
+			if tag&tagNotSafeToIgnoreMask != 0 {
+				return decodedBacking{}, errors.Newf("unknown tag %d", tag)
+			}
+			var dataLen uint64
+			dataLen, err = binary.ReadUvarint(br)
+			if err == nil {
+				_, err = br.Seek(int64(dataLen), io.SeekCurrent)
+			}
+		}
+		if err != nil {
+			return decodedBacking{}, err
+		}
+	}
+	if creatorID == 0 {
+		return decodedBacking{}, errors.Newf("shared object backing missing creator ID")
+	}
+	if creatorFileNum == 0 {
+		return decodedBacking{}, errors.Newf("shared object backing missing creator file num")
+	}
+	var res decodedBacking
+	res.meta.DiskFileNum = fileNum
+	res.meta.FileType = fileType
+	res.meta.Shared.CreatorID = objstorage.CreatorID(creatorID)
+	res.meta.Shared.CreatorFileNum = base.FileNum(creatorFileNum).DiskFileNum()
+	res.meta.Shared.CleanupMethod = objstorage.SharedCleanupMethod(cleanupMethod)
+
+	if res.meta.Shared.CleanupMethod == objstorage.SharedRefTracking {
+		if refCheckCreatorID == 0 || refCheckFileNum == 0 {
+			return decodedBacking{}, errors.Newf("shared object backing missing ref to check")
+		}
+		res.refToCheck.creatorID = objstorage.CreatorID(refCheckCreatorID)
+		res.refToCheck.fileNum = base.FileNum(refCheckFileNum).DiskFileNum()
+	}
+	return res, nil
+}
+
+// AttachSharedObjects is part of the objstorage.Provider interface.
+func (p *provider) AttachSharedObjects(
+	objs []objstorage.SharedObjectToAttach,
+) ([]objstorage.ObjectMetadata, error) {
+	decoded := make([]decodedBacking, len(objs))
+	for i, o := range objs {
+		var err error
+		decoded[i], err = decodeSharedObjectBacking(o.FileType, o.FileNum, o.Backing)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Create the reference marker objects.
+	// TODO(radu): parallelize this.
+	for _, d := range decoded {
+		if d.meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
+			continue
+		}
+		if err := p.sharedCreateRef(d.meta); err != nil {
+			// TODO(radu): clean up references previously created in this loop.
+			return nil, err
+		}
+		// Check the "origin"'s reference.
+		refName := sharedObjectRefName(d.meta, d.refToCheck.creatorID, d.refToCheck.fileNum)
+		if _, err := p.sharedStorage().Size(refName); err != nil {
+			_ = p.sharedUnref(d.meta)
+			// TODO(radu): clean up references previously created in this loop.
+			if p.sharedStorage().IsNotExistError(err) {
+				return nil, errors.Errorf("origin marker object %q does not exist;"+
+					" object probably removed from the provider which created the backing", refName)
+			}
+			return nil, errors.Wrapf(err, "checking origin's marker object %s", refName)
+		}
+	}
+
+	func() {
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		for _, d := range decoded {
+			p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
+				FileNum:        d.meta.DiskFileNum,
+				FileType:       d.meta.FileType,
+				CreatorID:      d.meta.Shared.CreatorID,
+				CreatorFileNum: d.meta.Shared.CreatorFileNum,
+			})
+		}
+	}()
+	if err := p.sharedSync(); err != nil {
+		return nil, err
+	}
+
+	metas := make([]objstorage.ObjectMetadata, len(decoded))
+	for i, d := range decoded {
+		metas[i] = d.meta
+	}
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	for _, meta := range metas {
+		p.mu.knownObjects[meta.DiskFileNum] = meta
+	}
+	return metas, nil
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_obj_name.go

@@ -0,0 +1,80 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"fmt"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+)
+
+// sharedObjectName returns the name of an object on shared storage.
+//
+// For sstables, the format is: <hash>-<creator-id>-<file-num>.sst
+// For example: 1a3f-2-000001.sst
+func sharedObjectName(meta objstorage.ObjectMetadata) string {
+	switch meta.FileType {
+	case base.FileTypeTable:
+		return fmt.Sprintf(
+			"%04x-%d-%06d.sst",
+			objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(),
+		)
+	}
+	panic("unknown FileType")
+}
+
+// sharedObjectRefName returns the name of the object's ref marker associated
+// with a given referencing provider. This name is the object's name concatenated with
+// ".ref.<ref-creator-id>.<local-file-num>".
+//
+// For example: 1a3f-2-000001.sst.ref.5.000008
+func sharedObjectRefName(
+	meta objstorage.ObjectMetadata, refCreatorID objstorage.CreatorID, refFileNum base.DiskFileNum,
+) string {
+	if meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
+		panic("ref object used when ref tracking disabled")
+	}
+	switch meta.FileType {
+	case base.FileTypeTable:
+		return fmt.Sprintf(
+			"%04x-%d-%06d.sst.ref.%d.%06d",
+			objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(), refCreatorID, refFileNum.FileNum(),
+		)
+	}
+	panic("unknown FileType")
+}
+
+func sharedObjectRefPrefix(meta objstorage.ObjectMetadata) string {
+	switch meta.FileType {
+	case base.FileTypeTable:
+		return fmt.Sprintf(
+			"%04x-%d-%06d.sst.ref.",
+			objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(),
+		)
+	}
+	panic("unknown FileType")
+}
+
+// sharedObjectRefName returns the name of the object's ref marker associated
+// with this provider. This name is the object's name concatenated with
+// ".ref.<creator-id>.<local-file-num>".
+//
+// For example: 1a3f-2-000001.sst.ref.5.000008
+func (p *provider) sharedObjectRefName(meta objstorage.ObjectMetadata) string {
+	if meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
+		panic("ref object used when ref tracking disabled")
+	}
+	return sharedObjectRefName(meta, p.shared.creatorID, meta.DiskFileNum)
+}
+
+// objHash returns a 16-bit hash value derived from the creator ID and creator
+// file num. We prepend this value to object names to ensure balanced
+// partitioning with AWS (and likely other blob storage providers).
+func objHash(meta objstorage.ObjectMetadata) uint16 {
+	const prime1 = 7459
+	const prime2 = 17539
+	return uint16(uint64(meta.Shared.CreatorID)*prime1 + uint64(meta.Shared.CreatorFileNum.FileNum())*prime2)
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_readable.go

@@ -0,0 +1,65 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/shared"
+)
+
+// sharedReadable is a very simple implementation of Readable on top of the
+// ReadCloser returned by shared.Storage.CreateObject.
+type sharedReadable struct {
+	objReader shared.ObjectReader
+	size      int64
+}
+
+var _ objstorage.Readable = (*sharedReadable)(nil)
+
+func newSharedReadable(objReader shared.ObjectReader, size int64) *sharedReadable {
+	return &sharedReadable{
+		objReader: objReader,
+		size:      size,
+	}
+}
+
+func (r *sharedReadable) ReadAt(ctx context.Context, p []byte, offset int64) error {
+	return r.objReader.ReadAt(ctx, p, offset)
+}
+
+func (r *sharedReadable) Close() error {
+	defer func() { r.objReader = nil }()
+	return r.objReader.Close()
+}
+
+func (r *sharedReadable) Size() int64 {
+	return r.size
+}
+
+func (r *sharedReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
+	// TODO(radu): use a pool.
+	return &sharedReadHandle{readable: r}
+}
+
+type sharedReadHandle struct {
+	readable *sharedReadable
+}
+
+var _ objstorage.ReadHandle = (*sharedReadHandle)(nil)
+
+func (r *sharedReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) error {
+	return r.readable.ReadAt(ctx, p, offset)
+}
+
+func (r *sharedReadHandle) Close() error {
+	r.readable = nil
+	return nil
+}
+
+func (r *sharedReadHandle) MaxReadahead() {}
+
+func (r *sharedReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_writable.go

@@ -0,0 +1,54 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"io"
+
+	"github.com/cockroachdb/pebble/objstorage"
+)
+
+// sharedWritable is a very simple implementation of Writable on top of the
+// WriteCloser returned by shared.Storage.CreateObject.
+type sharedWritable struct {
+	p             *provider
+	meta          objstorage.ObjectMetadata
+	storageWriter io.WriteCloser
+}
+
+var _ objstorage.Writable = (*sharedWritable)(nil)
+
+// Write is part of the Writable interface.
+func (w *sharedWritable) Write(p []byte) error {
+	_, err := w.storageWriter.Write(p)
+	return err
+}
+
+// Finish is part of the Writable interface.
+func (w *sharedWritable) Finish() error {
+	err := w.storageWriter.Close()
+	w.storageWriter = nil
+	if err != nil {
+		w.Abort()
+		return err
+	}
+
+	// Create the marker object.
+	if err := w.p.sharedCreateRef(w.meta); err != nil {
+		w.Abort()
+		return err
+	}
+	return nil
+}
+
+// Abort is part of the Writable interface.
+func (w *sharedWritable) Abort() {
+	if w.storageWriter != nil {
+		_ = w.storageWriter.Close()
+		w.storageWriter = nil
+	}
+	w.p.removeMetadata(w.meta.DiskFileNum)
+	// TODO(radu): delete the object if it was created.
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat/catalog.go

@@ -0,0 +1,376 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sharedobjcat
+
+import (
+	"fmt"
+	"io"
+	"sort"
+	"sync"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/record"
+	"github.com/cockroachdb/pebble/vfs"
+	"github.com/cockroachdb/pebble/vfs/atomicfs"
+)
+
+// Catalog is used to manage the on-disk shared object catalog.
+//
+// The catalog file is a log of records, where each record is an encoded
+// versionEdit.
+type Catalog struct {
+	fs      vfs.FS
+	dirname string
+	mu      struct {
+		sync.Mutex
+
+		creatorID objstorage.CreatorID
+		objects   map[base.DiskFileNum]SharedObjectMetadata
+
+		marker *atomicfs.Marker
+
+		catalogFile      vfs.File
+		catalogRecWriter *record.Writer
+
+		rotationHelper record.RotationHelper
+
+		// catalogFilename is the filename of catalogFile when catalogFile != nil, otherwise
+		// it is the filename of the last catalog file.
+		catalogFilename string
+	}
+}
+
+// SharedObjectMetadata encapsulates the data stored in the catalog file for each object.
+type SharedObjectMetadata struct {
+	// FileNum is the identifier for the object within the context of a single DB
+	// instance.
+	FileNum base.DiskFileNum
+	// FileType is the type of the object. Only certain FileTypes are possible.
+	FileType base.FileType
+	// CreatorID identifies the DB instance that originally created the object.
+	CreatorID objstorage.CreatorID
+	// CreatorFileNum is the identifier for the object within the context of the
+	// DB instance that originally created the object.
+	CreatorFileNum base.DiskFileNum
+
+	CleanupMethod objstorage.SharedCleanupMethod
+}
+
+const (
+	catalogFilenameBase = "SHARED-CATALOG"
+	catalogMarkerName   = "shared-catalog"
+
+	// We create a new file when the size exceeds 1MB (and some other conditions
+	// hold; see record.RotationHelper).
+	rotateFileSize = 1024 * 1024 // 1MB
+)
+
+// CatalogContents contains the shared objects in the catalog.
+type CatalogContents struct {
+	// CreatorID, if it is set.
+	CreatorID objstorage.CreatorID
+	Objects   []SharedObjectMetadata
+}
+
+// Open creates a Catalog and loads any existing catalog file, returning the
+// creator ID (if it is set) and the contents.
+func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
+	c := &Catalog{
+		fs:      fs,
+		dirname: dirname,
+	}
+	c.mu.objects = make(map[base.DiskFileNum]SharedObjectMetadata)
+
+	var err error
+	c.mu.marker, c.mu.catalogFilename, err = atomicfs.LocateMarker(fs, dirname, catalogMarkerName)
+	if err != nil {
+		return nil, CatalogContents{}, err
+	}
+	// If the filename is empty, there is no existing catalog.
+	if c.mu.catalogFilename != "" {
+		if err := c.loadFromCatalogFile(c.mu.catalogFilename); err != nil {
+			return nil, CatalogContents{}, err
+		}
+		if err := c.mu.marker.RemoveObsolete(); err != nil {
+			return nil, CatalogContents{}, err
+		}
+		// TODO(radu): remove obsolete catalog files.
+	}
+	res := CatalogContents{
+		CreatorID: c.mu.creatorID,
+		Objects:   make([]SharedObjectMetadata, 0, len(c.mu.objects)),
+	}
+	for _, meta := range c.mu.objects {
+		res.Objects = append(res.Objects, meta)
+	}
+	// Sort the objects so the function is deterministic.
+	sort.Slice(res.Objects, func(i, j int) bool {
+		return res.Objects[i].FileNum.FileNum() < res.Objects[j].FileNum.FileNum()
+	})
+	return c, res, nil
+}
+
+// SetCreatorID sets the creator ID. If it is already set, it must match.
+func (c *Catalog) SetCreatorID(id objstorage.CreatorID) error {
+	if !id.IsSet() {
+		return errors.AssertionFailedf("attempt to unset CreatorID")
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.mu.creatorID.IsSet() {
+		if c.mu.creatorID != id {
+			return errors.AssertionFailedf("attempt to change CreatorID from %s to %s", c.mu.creatorID, id)
+		}
+		return nil
+	}
+
+	ve := versionEdit{CreatorID: id}
+	if err := c.writeToCatalogFileLocked(&ve); err != nil {
+		return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
+	}
+	c.mu.creatorID = id
+	return nil
+}
+
+// Close any open files.
+func (c *Catalog) Close() error {
+	return c.closeCatalogFile()
+}
+
+func (c *Catalog) closeCatalogFile() error {
+	if c.mu.catalogFile == nil {
+		return nil
+	}
+	err1 := c.mu.catalogRecWriter.Close()
+	err2 := c.mu.catalogFile.Close()
+	c.mu.catalogRecWriter = nil
+	c.mu.catalogFile = nil
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+// Batch is used to perform multiple object additions/deletions at once.
+type Batch struct {
+	ve versionEdit
+}
+
+// AddObject adds a new object to the batch.
+//
+// The given FileNum must be new - it must not match that of any object that was
+// ever in the catalog.
+func (b *Batch) AddObject(meta SharedObjectMetadata) {
+	b.ve.NewObjects = append(b.ve.NewObjects, meta)
+}
+
+// DeleteObject adds an object removal to the batch.
+func (b *Batch) DeleteObject(fileNum base.DiskFileNum) {
+	b.ve.DeletedObjects = append(b.ve.DeletedObjects, fileNum)
+}
+
+// Reset clears the batch.
+func (b *Batch) Reset() {
+	b.ve.NewObjects = b.ve.NewObjects[:0]
+	b.ve.DeletedObjects = b.ve.DeletedObjects[:0]
+}
+
+// IsEmpty returns true if the batch is empty.
+func (b *Batch) IsEmpty() bool {
+	return len(b.ve.NewObjects) == 0 && len(b.ve.DeletedObjects) == 0
+}
+
+// Copy returns a copy of the Batch.
+func (b *Batch) Copy() Batch {
+	var res Batch
+	if len(b.ve.NewObjects) > 0 {
+		res.ve.NewObjects = make([]SharedObjectMetadata, len(b.ve.NewObjects))
+		copy(res.ve.NewObjects, b.ve.NewObjects)
+	}
+	if len(b.ve.DeletedObjects) > 0 {
+		res.ve.DeletedObjects = make([]base.DiskFileNum, len(b.ve.DeletedObjects))
+		copy(res.ve.DeletedObjects, b.ve.DeletedObjects)
+	}
+	return res
+}
+
+// Append merges two batches.
+func (b *Batch) Append(other Batch) {
+	b.ve.NewObjects = append(b.ve.NewObjects, other.ve.NewObjects...)
+	b.ve.DeletedObjects = append(b.ve.DeletedObjects, other.ve.DeletedObjects...)
+}
+
+// ApplyBatch applies a batch of updates; returns after the change is stably
+// recorded on storage.
+func (c *Catalog) ApplyBatch(b Batch) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	for _, n := range b.ve.DeletedObjects {
+		if _, exists := c.mu.objects[n]; !exists {
+			return errors.AssertionFailedf("deleting non-existent object %s", n)
+		}
+	}
+	for _, meta := range b.ve.NewObjects {
+		if _, exists := c.mu.objects[meta.FileNum]; exists {
+			return errors.AssertionFailedf("adding existing object %s", meta.FileNum)
+		}
+	}
+
+	if err := c.writeToCatalogFileLocked(&b.ve); err != nil {
+		return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
+	}
+
+	// Apply the batch to our current state.
+	for _, n := range b.ve.DeletedObjects {
+		delete(c.mu.objects, n)
+	}
+	for _, meta := range b.ve.NewObjects {
+		c.mu.objects[meta.FileNum] = meta
+	}
+	b.Reset()
+	return nil
+}
+
+func (c *Catalog) loadFromCatalogFile(filename string) error {
+	catalogPath := c.fs.PathJoin(c.dirname, filename)
+	f, err := c.fs.Open(catalogPath)
+	if err != nil {
+		return errors.Wrapf(
+			err, "pebble: could not open shared object catalog file %q for DB %q",
+			errors.Safe(filename), c.dirname,
+		)
+	}
+	defer f.Close()
+	rr := record.NewReader(f, 0 /* logNum */)
+	for {
+		r, err := rr.Next()
+		if err == io.EOF || record.IsInvalidRecord(err) {
+			break
+		}
+		if err != nil {
+			return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
+				errors.Safe(filename))
+		}
+		var ve versionEdit
+		err = ve.Decode(r)
+		if err != nil {
+			return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
+				errors.Safe(filename))
+		}
+		// Apply the version edit to the current state.
+		if ve.CreatorID.IsSet() {
+			c.mu.creatorID = ve.CreatorID
+		}
+		for _, fileNum := range ve.DeletedObjects {
+			delete(c.mu.objects, fileNum)
+		}
+		for _, meta := range ve.NewObjects {
+			c.mu.objects[meta.FileNum] = meta
+		}
+	}
+	return nil
+}
+
+// writeToCatalogFileLocked writes a versionEdit to the catalog file.
+// Creates a new file if this is the first write.
+func (c *Catalog) writeToCatalogFileLocked(ve *versionEdit) error {
+	c.mu.rotationHelper.AddRecord(int64(len(ve.NewObjects) + len(ve.DeletedObjects)))
+	snapshotSize := int64(len(c.mu.objects))
+
+	var shouldRotate bool
+	if c.mu.catalogFile == nil {
+		shouldRotate = true
+	} else if c.mu.catalogRecWriter.Size() >= rotateFileSize {
+		shouldRotate = c.mu.rotationHelper.ShouldRotate(snapshotSize)
+	}
+
+	if shouldRotate {
+		if c.mu.catalogFile != nil {
+			if err := c.closeCatalogFile(); err != nil {
+				return err
+			}
+		}
+		if err := c.createNewCatalogFileLocked(); err != nil {
+			return err
+		}
+		c.mu.rotationHelper.Rotate(snapshotSize)
+	}
+	return writeRecord(ve, c.mu.catalogFile, c.mu.catalogRecWriter)
+}
+
+func makeCatalogFilename(iter uint64) string {
+	return fmt.Sprintf("%s-%06d", catalogFilenameBase, iter)
+}
+
+// createNewCatalogFileLocked creates a new catalog file, populates it with the
+// current catalog and sets c.mu.catalogFile and c.mu.catalogRecWriter.
+func (c *Catalog) createNewCatalogFileLocked() (outErr error) {
+	if c.mu.catalogFile != nil {
+		return errors.AssertionFailedf("catalogFile already open")
+	}
+	filename := makeCatalogFilename(c.mu.marker.NextIter())
+	filepath := c.fs.PathJoin(c.dirname, filename)
+	file, err := c.fs.Create(filepath)
+	if err != nil {
+		return err
+	}
+	recWriter := record.NewWriter(file)
+	err = func() error {
+		// Create a versionEdit that gets us from an empty catalog to the current state.
+		var ve versionEdit
+		ve.CreatorID = c.mu.creatorID
+		ve.NewObjects = make([]SharedObjectMetadata, 0, len(c.mu.objects))
+		for _, meta := range c.mu.objects {
+			ve.NewObjects = append(ve.NewObjects, meta)
+		}
+		if err := writeRecord(&ve, file, recWriter); err != nil {
+			return err
+		}
+
+		// Move the marker to the new filename. Move handles syncing the data
+		// directory as well.
+		if err := c.mu.marker.Move(filename); err != nil {
+			return errors.Wrap(err, "moving marker")
+		}
+
+		return nil
+	}()
+
+	if err != nil {
+		_ = recWriter.Close()
+		_ = file.Close()
+		_ = c.fs.Remove(filepath)
+		return err
+	}
+
+	// Remove any previous file (ignoring any error).
+	if c.mu.catalogFilename != "" {
+		_ = c.fs.Remove(c.fs.PathJoin(c.dirname, c.mu.catalogFilename))
+	}
+
+	c.mu.catalogFile = file
+	c.mu.catalogRecWriter = recWriter
+	c.mu.catalogFilename = filename
+	return nil
+}
+
+func writeRecord(ve *versionEdit, file vfs.File, recWriter *record.Writer) error {
+	w, err := recWriter.Next()
+	if err != nil {
+		return err
+	}
+	if err := ve.Encode(w); err != nil {
+		return err
+	}
+	if err := recWriter.Flush(); err != nil {
+		return err
+	}
+	return file.Sync()
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat/version_edit.go

@@ -0,0 +1,165 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sharedobjcat
+
+import (
+	"bufio"
+	"encoding/binary"
+	"io"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+)
+
+// versionEdit is a modification to the shared object state which can be encoded
+// into a record.
+//
+// TODO(radu): consider adding creation and deletion time for debugging purposes.
+type versionEdit struct {
+	NewObjects     []SharedObjectMetadata
+	DeletedObjects []base.DiskFileNum
+	CreatorID      objstorage.CreatorID
+}
+
+const (
+	// tagNewObject is followed by the FileNum, creator ID, creator FileNum, and
+	// cleanup method.
+	tagNewObject = 1
+	// tagDeletedObject is followed by the FileNum.
+	tagDeletedObject = 2
+	// tagCreatorID is followed by the Creator ID for this store. This ID can
+	// never change.
+	tagCreatorID = 3
+)
+
+// Object type values. We don't want to encode FileType directly because it is
+// more general (and we want freedom to change it in the future).
+const (
+	objTypeTable = 1
+)
+
+func objTypeToFileType(objType uint64) (base.FileType, error) {
+	switch objType {
+	case objTypeTable:
+		return base.FileTypeTable, nil
+	default:
+		return 0, errors.Newf("unknown object type %d", objType)
+	}
+}
+
+func fileTypeToObjType(fileType base.FileType) (uint64, error) {
+	switch fileType {
+	case base.FileTypeTable:
+		return objTypeTable, nil
+
+	default:
+		return 0, errors.Newf("unknown object type for file type %d", fileType)
+	}
+}
+
+// Encode encodes an edit to the specified writer.
+func (v *versionEdit) Encode(w io.Writer) error {
+	buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*4+len(v.DeletedObjects)*2+2))
+	for _, meta := range v.NewObjects {
+		objType, err := fileTypeToObjType(meta.FileType)
+		if err != nil {
+			return err
+		}
+		buf = binary.AppendUvarint(buf, uint64(tagNewObject))
+		buf = binary.AppendUvarint(buf, uint64(meta.FileNum.FileNum()))
+		buf = binary.AppendUvarint(buf, objType)
+		buf = binary.AppendUvarint(buf, uint64(meta.CreatorID))
+		buf = binary.AppendUvarint(buf, uint64(meta.CreatorFileNum.FileNum()))
+		buf = binary.AppendUvarint(buf, uint64(meta.CleanupMethod))
+	}
+
+	for _, dfn := range v.DeletedObjects {
+		buf = binary.AppendUvarint(buf, uint64(tagDeletedObject))
+		buf = binary.AppendUvarint(buf, uint64(dfn.FileNum()))
+	}
+	if v.CreatorID.IsSet() {
+		buf = binary.AppendUvarint(buf, uint64(tagCreatorID))
+		buf = binary.AppendUvarint(buf, uint64(v.CreatorID))
+	}
+	_, err := w.Write(buf)
+	return err
+}
+
+// Decode decodes an edit from the specified reader.
+func (v *versionEdit) Decode(r io.Reader) error {
+	br, ok := r.(io.ByteReader)
+	if !ok {
+		br = bufio.NewReader(r)
+	}
+	for {
+		tag, err := binary.ReadUvarint(br)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		err = nil
+		switch tag {
+		case tagNewObject:
+			var fileNum, creatorID, creatorFileNum, cleanupMethod uint64
+			var fileType base.FileType
+			fileNum, err = binary.ReadUvarint(br)
+			if err == nil {
+				var objType uint64
+				objType, err = binary.ReadUvarint(br)
+				if err == nil {
+					fileType, err = objTypeToFileType(objType)
+				}
+			}
+			if err == nil {
+				creatorID, err = binary.ReadUvarint(br)
+			}
+			if err == nil {
+				creatorFileNum, err = binary.ReadUvarint(br)
+			}
+			if err == nil {
+				cleanupMethod, err = binary.ReadUvarint(br)
+			}
+			if err == nil {
+				v.NewObjects = append(v.NewObjects, SharedObjectMetadata{
+					FileNum:        base.FileNum(fileNum).DiskFileNum(),
+					FileType:       fileType,
+					CreatorID:      objstorage.CreatorID(creatorID),
+					CreatorFileNum: base.FileNum(creatorFileNum).DiskFileNum(),
+					CleanupMethod:  objstorage.SharedCleanupMethod(cleanupMethod),
+				})
+			}
+
+		case tagDeletedObject:
+			var fileNum uint64
+			fileNum, err = binary.ReadUvarint(br)
+			if err == nil {
+				v.DeletedObjects = append(v.DeletedObjects, base.FileNum(fileNum).DiskFileNum())
+			}
+
+		case tagCreatorID:
+			var id uint64
+			id, err = binary.ReadUvarint(br)
+			if err == nil {
+				v.CreatorID = objstorage.CreatorID(id)
+			}
+
+		default:
+			// Ignore unknown tags.
+		}
+		if err != nil {
+			if err == io.EOF {
+				return errCorruptCatalog
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt shared object catalog")

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs.go

@@ -0,0 +1,109 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+func (p *provider) vfsPath(fileType base.FileType, fileNum base.DiskFileNum) string {
+	return base.MakeFilepath(p.st.FS, p.st.FSDirName, fileType, fileNum)
+}
+
+func (p *provider) vfsOpenForReading(
+	ctx context.Context,
+	fileType base.FileType,
+	fileNum base.DiskFileNum,
+	opts objstorage.OpenOptions,
+) (objstorage.Readable, error) {
+	filename := p.vfsPath(fileType, fileNum)
+	file, err := p.st.FS.Open(filename, vfs.RandomReadsOption)
+	if err != nil {
+		if opts.MustExist {
+			base.MustExist(p.st.FS, filename, p.st.Logger, err)
+		}
+		return nil, err
+	}
+	return newFileReadable(file, p.st.FS, filename)
+}
+
+func (p *provider) vfsCreate(
+	_ context.Context, fileType base.FileType, fileNum base.DiskFileNum,
+) (objstorage.Writable, objstorage.ObjectMetadata, error) {
+	filename := p.vfsPath(fileType, fileNum)
+	file, err := p.st.FS.Create(filename)
+	if err != nil {
+		return nil, objstorage.ObjectMetadata{}, err
+	}
+	file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{
+		NoSyncOnClose: p.st.NoSyncOnClose,
+		BytesPerSync:  p.st.BytesPerSync,
+	})
+	meta := objstorage.ObjectMetadata{
+		DiskFileNum: fileNum,
+		FileType:    fileType,
+	}
+	return newFileBufferedWritable(file), meta, nil
+}
+
+func (p *provider) vfsRemove(fileType base.FileType, fileNum base.DiskFileNum) error {
+	return p.st.FSCleaner.Clean(p.st.FS, fileType, p.vfsPath(fileType, fileNum))
+}
+
+// vfsInit finds any local FS objects.
+func (p *provider) vfsInit() error {
+	listing := p.st.FSDirInitialListing
+	if listing == nil {
+		var err error
+		listing, err = p.st.FS.List(p.st.FSDirName)
+		if err != nil {
+			return errors.Wrapf(err, "pebble: could not list store directory")
+		}
+	}
+
+	for _, filename := range listing {
+		fileType, fileNum, ok := base.ParseFilename(p.st.FS, filename)
+		if ok && fileType == base.FileTypeTable {
+			o := objstorage.ObjectMetadata{
+				FileType:    fileType,
+				DiskFileNum: fileNum,
+			}
+			p.mu.knownObjects[o.DiskFileNum] = o
+		}
+	}
+	return nil
+}
+
+func (p *provider) vfsSync() error {
+	p.mu.Lock()
+	shouldSync := p.mu.localObjectsChanged
+	p.mu.localObjectsChanged = false
+	p.mu.Unlock()
+
+	if !shouldSync {
+		return nil
+	}
+	if err := p.fsDir.Sync(); err != nil {
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		p.mu.localObjectsChanged = true
+		return err
+	}
+	return nil
+}
+
+func (p *provider) vfsSize(fileType base.FileType, fileNum base.DiskFileNum) (int64, error) {
+	filename := p.vfsPath(fileType, fileNum)
+	stat, err := p.st.FS.Stat(filename)
+	if err != nil {
+		return 0, err
+	}
+	return stat.Size(), nil
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go

@@ -0,0 +1,210 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// fileReadable implements objstorage.Readable on top of a vfs.File.
+//
+// The implementation might use Prealloc and might reopen the file with
+// SequentialReadsOption.
+type fileReadable struct {
+	file vfs.File
+	size int64
+
+	// The following fields are used to possibly open the file again using the
+	// sequential reads option (see vfsReadHandle).
+	filename string
+	fs       vfs.FS
+}
+
+var _ objstorage.Readable = (*fileReadable)(nil)
+
+func newFileReadable(file vfs.File, fs vfs.FS, filename string) (*fileReadable, error) {
+	info, err := file.Stat()
+	if err != nil {
+		return nil, err
+	}
+	r := &fileReadable{
+		file:     file,
+		size:     info.Size(),
+		filename: filename,
+		fs:       fs,
+	}
+	invariants.SetFinalizer(r, func(obj interface{}) {
+		if obj.(*fileReadable).file != nil {
+			fmt.Fprintf(os.Stderr, "Readable was not closed")
+			os.Exit(1)
+		}
+	})
+	return r, nil
+}
+
+// ReadAt is part of the objstorage.Readable interface.
+func (r *fileReadable) ReadAt(_ context.Context, p []byte, off int64) error {
+	n, err := r.file.ReadAt(p, off)
+	if invariants.Enabled && err == nil && n != len(p) {
+		panic("short read")
+	}
+	return err
+}
+
+// Close is part of the objstorage.Readable interface.
+func (r *fileReadable) Close() error {
+	defer func() { r.file = nil }()
+	return r.file.Close()
+}
+
+// Size is part of the objstorage.Readable interface.
+func (r *fileReadable) Size() int64 {
+	return r.size
+}
+
+// NewReadHandle is part of the objstorage.Readable interface.
+func (r *fileReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
+	rh := readHandlePool.Get().(*vfsReadHandle)
+	rh.r = r
+	rh.rs = makeReadaheadState()
+	return rh
+}
+
+type vfsReadHandle struct {
+	r  *fileReadable
+	rs readaheadState
+
+	// sequentialFile holds a file descriptor to the same underlying File,
+	// except with fadvise(FADV_SEQUENTIAL) called on it to take advantage of
+	// OS-level readahead. Once this is non-nil, the other variables in
+	// readaheadState don't matter much as we defer to OS-level readahead.
+	sequentialFile vfs.File
+}
+
+var _ objstorage.ReadHandle = (*vfsReadHandle)(nil)
+
+var readHandlePool = sync.Pool{
+	New: func() interface{} {
+		i := &vfsReadHandle{}
+		// Note: this is a no-op if invariants are disabled or race is enabled.
+		invariants.SetFinalizer(i, func(obj interface{}) {
+			if obj.(*vfsReadHandle).r != nil {
+				fmt.Fprintf(os.Stderr, "ReadHandle was not closed")
+				os.Exit(1)
+			}
+		})
+		return i
+	},
+}
+
+// Close is part of the objstorage.ReadHandle interface.
+func (rh *vfsReadHandle) Close() error {
+	var err error
+	if rh.sequentialFile != nil {
+		err = rh.sequentialFile.Close()
+	}
+	*rh = vfsReadHandle{}
+	readHandlePool.Put(rh)
+	return err
+}
+
+// ReadAt is part of the objstorage.ReadHandle interface.
+func (rh *vfsReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error {
+	var n int
+	var err error
+	if rh.sequentialFile != nil {
+		// Use OS-level read-ahead.
+		n, err = rh.sequentialFile.ReadAt(p, offset)
+	} else {
+		if readaheadSize := rh.rs.maybeReadahead(offset, int64(len(p))); readaheadSize > 0 {
+			if readaheadSize >= maxReadaheadSize {
+				// We've reached the maximum readahead size. Beyond this point, rely on
+				// OS-level readahead.
+				rh.MaxReadahead()
+			} else {
+				_ = rh.r.file.Prefetch(offset, readaheadSize)
+			}
+		}
+		n, err = rh.r.file.ReadAt(p, offset)
+	}
+	if invariants.Enabled && err == nil && n != len(p) {
+		panic("short read")
+	}
+	return err
+}
+
+// MaxReadahead is part of the objstorage.ReadHandle interface.
+func (rh *vfsReadHandle) MaxReadahead() {
+	if rh.sequentialFile != nil {
+		return
+	}
+
+	// TODO(radu): we could share the reopened file descriptor across multiple
+	// handles.
+	f, err := rh.r.fs.Open(rh.r.filename, vfs.SequentialReadsOption)
+	if err == nil {
+		rh.sequentialFile = f
+	}
+}
+
+// RecordCacheHit is part of the objstorage.ReadHandle interface.
+func (rh *vfsReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
+	if rh.sequentialFile != nil {
+		// Using OS-level readahead, so do nothing.
+		return
+	}
+	rh.rs.recordCacheHit(offset, size)
+}
+
+// TestingCheckMaxReadahead returns true if the ReadHandle has switched to
+// OS-level read-ahead.
+func TestingCheckMaxReadahead(rh objstorage.ReadHandle) bool {
+	switch rh := rh.(type) {
+	case *vfsReadHandle:
+		return rh.sequentialFile != nil
+	case *PreallocatedReadHandle:
+		return rh.sequentialFile != nil
+	default:
+		panic("unknown ReadHandle type")
+	}
+}
+
+// PreallocatedReadHandle is used to avoid an allocation in NewReadHandle; see
+// UsePreallocatedReadHandle.
+type PreallocatedReadHandle struct {
+	vfsReadHandle
+}
+
+// Close is part of the objstorage.ReadHandle interface.
+func (rh *PreallocatedReadHandle) Close() error {
+	var err error
+	if rh.sequentialFile != nil {
+		err = rh.sequentialFile.Close()
+	}
+	rh.vfsReadHandle = vfsReadHandle{}
+	return err
+}
+
+// UsePreallocatedReadHandle is equivalent to calling readable.NewReadHandle()
+// but uses the existing storage of a PreallocatedReadHandle when possible
+// (currently this happens if we are reading from a local file).
+// The returned handle still needs to be closed.
+func UsePreallocatedReadHandle(
+	ctx context.Context, readable objstorage.Readable, rh *PreallocatedReadHandle,
+) objstorage.ReadHandle {
+	if r, ok := readable.(*fileReadable); ok {
+		// See fileReadable.NewReadHandle.
+		rh.vfsReadHandle = vfsReadHandle{r: r}
+		return rh
+	}
+	return readable.NewReadHandle(ctx)
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_writable.go

@@ -0,0 +1,65 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"bufio"
+
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// NewFileWritable returns a Writable that uses a file as underlying storage.
+func NewFileWritable(file vfs.File) objstorage.Writable {
+	return newFileBufferedWritable(file)
+}
+
+type fileBufferedWritable struct {
+	file vfs.File
+	bw   *bufio.Writer
+}
+
+var _ objstorage.Writable = (*fileBufferedWritable)(nil)
+
+func newFileBufferedWritable(file vfs.File) *fileBufferedWritable {
+	return &fileBufferedWritable{
+		file: file,
+		bw:   bufio.NewWriter(file),
+	}
+}
+
+// Write is part of the objstorage.Writable interface.
+func (w *fileBufferedWritable) Write(p []byte) error {
+	// Ignoring the length written since bufio.Writer.Write is guaranteed to
+	// return an error if the length written is < len(p).
+	_, err := w.bw.Write(p)
+	return err
+}
+
+// Finish is part of the objstorage.Writable interface.
+func (w *fileBufferedWritable) Finish() error {
+	err := w.bw.Flush()
+	if err == nil {
+		err = w.file.Sync()
+	}
+	err = firstError(err, w.file.Close())
+	w.bw = nil
+	w.file = nil
+	return err
+}
+
+// Abort is part of the objstorage.Writable interface.
+func (w *fileBufferedWritable) Abort() {
+	_ = w.file.Close()
+	w.bw = nil
+	w.file = nil
+}
+
+func firstError(err0, err1 error) error {
+	if err0 != nil {
+		return err0
+	}
+	return err1
+}