[WIP] lint a bit, update deps

vendor/github.com/cockroachdb/pebble/objstorage/remote/factory.go

@@ -0,0 +1,25 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package remote
+
+import "github.com/pkg/errors"
+
+// MakeSimpleFactory returns a StorageFactory implementation that produces the given
+// Storage objects.
+func MakeSimpleFactory(m map[Locator]Storage) StorageFactory {
+	return simpleFactory(m)
+}
+
+type simpleFactory map[Locator]Storage
+
+var _ StorageFactory = simpleFactory{}
+
+// CreateStorage is part of the StorageFactory interface.
+func (sf simpleFactory) CreateStorage(locator Locator) (Storage, error) {
+	if s, ok := sf[locator]; ok {
+		return s, nil
+	}
+	return nil, errors.Errorf("unknown locator '%s'", locator)
+}

vendor/github.com/cockroachdb/pebble/objstorage/remote/localfs.go

@@ -0,0 +1,118 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package remote
+
+import (
+	"context"
+	"io"
+	"os"
+	"path"
+
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// NewLocalFS returns a vfs-backed implementation of the remote.Storage
+// interface (for testing). All objects will be stored at the directory
+// dirname.
+func NewLocalFS(dirname string, fs vfs.FS) Storage {
+	store := &localFSStore{
+		dirname: dirname,
+		vfs:     fs,
+	}
+	return store
+}
+
+// localFSStore is a vfs-backed implementation of the remote.Storage
+// interface (for testing).
+type localFSStore struct {
+	dirname string
+	vfs     vfs.FS
+}
+
+var _ Storage = (*localFSStore)(nil)
+
+// Close is part of the remote.Storage interface.
+func (s *localFSStore) Close() error {
+	*s = localFSStore{}
+	return nil
+}
+
+// ReadObject is part of the remote.Storage interface.
+func (s *localFSStore) ReadObject(
+	ctx context.Context, objName string,
+) (_ ObjectReader, objSize int64, _ error) {
+	f, err := s.vfs.Open(path.Join(s.dirname, objName))
+	if err != nil {
+		return nil, 0, err
+	}
+	stat, err := f.Stat()
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return &localFSReader{f}, stat.Size(), nil
+}
+
+type localFSReader struct {
+	file vfs.File
+}
+
+var _ ObjectReader = (*localFSReader)(nil)
+
+// ReadAt is part of the shared.ObjectReader interface.
+func (r *localFSReader) ReadAt(_ context.Context, p []byte, offset int64) error {
+	n, err := r.file.ReadAt(p, offset)
+	// https://pkg.go.dev/io#ReaderAt
+	if err == io.EOF && n == len(p) {
+		return nil
+	}
+	return err
+}
+
+// Close is part of the shared.ObjectReader interface.
+func (r *localFSReader) Close() error {
+	r.file.Close()
+	r.file = nil
+	return nil
+}
+
+// CreateObject is part of the remote.Storage interface.
+func (s *localFSStore) CreateObject(objName string) (io.WriteCloser, error) {
+	file, err := s.vfs.Create(path.Join(s.dirname, objName))
+	return file, err
+}
+
+// List is part of the remote.Storage interface.
+func (s *localFSStore) List(prefix, delimiter string) ([]string, error) {
+	// TODO(josh): For the intended use case of localfs.go of running 'pebble bench',
+	// List can always return <nil, nil>, since this indicates a file has only one ref,
+	// and since `pebble bench` implies running in a single-pebble-instance context.
+	// https://github.com/cockroachdb/pebble/blob/a9a079d4fb6bf4a9ebc52e4d83a76ad4cbf676cb/objstorage/objstorageprovider/shared.go#L292
+	return nil, nil
+}
+
+// Delete is part of the remote.Storage interface.
+func (s *localFSStore) Delete(objName string) error {
+	return s.vfs.Remove(path.Join(s.dirname, objName))
+}
+
+// Size is part of the remote.Storage interface.
+func (s *localFSStore) Size(objName string) (int64, error) {
+	f, err := s.vfs.Open(path.Join(s.dirname, objName))
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	stat, err := f.Stat()
+	if err != nil {
+		return 0, err
+	}
+	return stat.Size(), nil
+}
+
+// IsNotExistError is part of the remote.Storage interface.
+func (s *localFSStore) IsNotExistError(err error) bool {
+	return err == os.ErrNotExist
+}

vendor/github.com/cockroachdb/pebble/objstorage/remote/logging.go

@@ -0,0 +1,139 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package remote
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sort"
+)
+
+// WithLogging wraps the given Storage implementation and emits logs for various
+// operations.
+func WithLogging(wrapped Storage, logf func(fmt string, args ...interface{})) Storage {
+	return &loggingStore{
+		logf:    logf,
+		wrapped: wrapped,
+	}
+}
+
+// loggingStore wraps a remote.Storage implementation and emits logs of the
+// operations.
+type loggingStore struct {
+	logf    func(fmt string, args ...interface{})
+	wrapped Storage
+}
+
+var _ Storage = (*loggingStore)(nil)
+
+func (l *loggingStore) Close() error {
+	l.logf("close")
+	return l.wrapped.Close()
+}
+
+func (l *loggingStore) ReadObject(
+	ctx context.Context, objName string,
+) (_ ObjectReader, objSize int64, _ error) {
+	r, size, err := l.wrapped.ReadObject(ctx, objName)
+	l.logf("create reader for object %q: %s", objName, errOrPrintf(err, "%d bytes", size))
+	if err != nil {
+		return nil, 0, err
+	}
+	return &loggingReader{
+		l:       l,
+		name:    objName,
+		wrapped: r,
+	}, size, nil
+}
+
+type loggingReader struct {
+	l       *loggingStore
+	name    string
+	wrapped ObjectReader
+}
+
+var _ ObjectReader = (*loggingReader)(nil)
+
+func (l *loggingReader) ReadAt(ctx context.Context, p []byte, offset int64) error {
+	if err := l.wrapped.ReadAt(ctx, p, offset); err != nil {
+		l.l.logf("read object %q at %d (length %d): error %v", l.name, offset, len(p), err)
+		return err
+	}
+	l.l.logf("read object %q at %d (length %d)", l.name, offset, len(p))
+	return nil
+}
+
+func (l *loggingReader) Close() error {
+	l.l.logf("close reader for %q", l.name)
+	return l.wrapped.Close()
+}
+
+func (l *loggingStore) CreateObject(objName string) (io.WriteCloser, error) {
+	l.logf("create object %q", objName)
+	writer, err := l.wrapped.CreateObject(objName)
+	if err != nil {
+		return nil, err
+	}
+	return &loggingWriter{
+		l:           l,
+		name:        objName,
+		WriteCloser: writer,
+	}, nil
+}
+
+type loggingWriter struct {
+	l            *loggingStore
+	name         string
+	bytesWritten int64
+	io.WriteCloser
+}
+
+func (l *loggingWriter) Write(p []byte) (n int, err error) {
+	n, err = l.WriteCloser.Write(p)
+	l.bytesWritten += int64(n)
+	return n, err
+}
+
+func (l *loggingWriter) Close() error {
+	l.l.logf("close writer for %q after %d bytes", l.name, l.bytesWritten)
+	return l.WriteCloser.Close()
+}
+
+func (l *loggingStore) List(prefix, delimiter string) ([]string, error) {
+	l.logf("list (prefix=%q, delimiter=%q)", prefix, delimiter)
+	res, err := l.wrapped.List(prefix, delimiter)
+	if err != nil {
+		return nil, err
+	}
+	sorted := append([]string(nil), res...)
+	sort.Strings(sorted)
+	for _, s := range sorted {
+		l.logf(" - %s", s)
+	}
+	return res, nil
+}
+
+func (l *loggingStore) Delete(objName string) error {
+	l.logf("delete object %q", objName)
+	return l.wrapped.Delete(objName)
+}
+
+func (l *loggingStore) Size(objName string) (int64, error) {
+	size, err := l.wrapped.Size(objName)
+	l.logf("size of object %q: %s", objName, errOrPrintf(err, "%d", size))
+	return size, err
+}
+
+func errOrPrintf(err error, format string, args ...interface{}) string {
+	if err != nil {
+		return fmt.Sprintf("error: %v", err)
+	}
+	return fmt.Sprintf(format, args...)
+}
+
+func (l *loggingStore) IsNotExistError(err error) bool {
+	return l.wrapped.IsNotExistError(err)
+}

vendor/github.com/cockroachdb/pebble/objstorage/remote/mem.go

@@ -0,0 +1,161 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package remote
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"os"
+	"strings"
+	"sync"
+)
+
+// NewInMem returns an in-memory implementation of the remote.Storage
+// interface (for testing).
+func NewInMem() Storage {
+	store := &inMemStore{}
+	store.mu.objects = make(map[string]*inMemObj)
+	return store
+}
+
+// inMemStore is an in-memory implementation of the remote.Storage interface
+// (for testing).
+type inMemStore struct {
+	mu struct {
+		sync.Mutex
+		objects map[string]*inMemObj
+	}
+}
+
+var _ Storage = (*inMemStore)(nil)
+
+type inMemObj struct {
+	name string
+	data []byte
+}
+
+func (s *inMemStore) Close() error {
+	*s = inMemStore{}
+	return nil
+}
+
+func (s *inMemStore) ReadObject(
+	ctx context.Context, objName string,
+) (_ ObjectReader, objSize int64, _ error) {
+	obj, err := s.getObj(objName)
+	if err != nil {
+		return nil, 0, err
+	}
+	return &inMemReader{data: obj.data}, int64(len(obj.data)), nil
+}
+
+type inMemReader struct {
+	data []byte
+}
+
+var _ ObjectReader = (*inMemReader)(nil)
+
+func (r *inMemReader) ReadAt(ctx context.Context, p []byte, offset int64) error {
+	if offset+int64(len(p)) > int64(len(r.data)) {
+		return io.EOF
+	}
+	copy(p, r.data[offset:])
+	return nil
+}
+
+func (r *inMemReader) Close() error {
+	r.data = nil
+	return nil
+}
+
+func (s *inMemStore) CreateObject(objName string) (io.WriteCloser, error) {
+	return &inMemWriter{
+		store: s,
+		name:  objName,
+	}, nil
+}
+
+type inMemWriter struct {
+	store *inMemStore
+	name  string
+	buf   bytes.Buffer
+}
+
+var _ io.WriteCloser = (*inMemWriter)(nil)
+
+func (o *inMemWriter) Write(p []byte) (n int, err error) {
+	if o.store == nil {
+		panic("Write after Close")
+	}
+	return o.buf.Write(p)
+}
+
+func (o *inMemWriter) Close() error {
+	if o.store != nil {
+		o.store.addObj(&inMemObj{
+			name: o.name,
+			data: o.buf.Bytes(),
+		})
+		o.store = nil
+	}
+	return nil
+}
+
+func (s *inMemStore) List(prefix, delimiter string) ([]string, error) {
+	if delimiter != "" {
+		panic("delimiter unimplemented")
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	res := make([]string, 0, len(s.mu.objects))
+	for name := range s.mu.objects {
+		if strings.HasPrefix(name, prefix) {
+			res = append(res, name)
+		}
+	}
+	return res, nil
+}
+
+func (s *inMemStore) Delete(objName string) error {
+	s.rmObj(objName)
+	return nil
+}
+
+// Size returns the length of the named object in bytesWritten.
+func (s *inMemStore) Size(objName string) (int64, error) {
+	obj, err := s.getObj(objName)
+	if err != nil {
+		return 0, err
+	}
+	return int64(len(obj.data)), nil
+}
+
+func (s *inMemStore) IsNotExistError(err error) bool {
+	return err == os.ErrNotExist
+}
+
+func (s *inMemStore) getObj(name string) (*inMemObj, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	obj, ok := s.mu.objects[name]
+	if !ok {
+		return nil, os.ErrNotExist
+	}
+	return obj, nil
+}
+
+func (s *inMemStore) addObj(o *inMemObj) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.mu.objects[o.name] = o
+}
+
+func (s *inMemStore) rmObj(name string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	delete(s.mu.objects, name)
+}

vendor/github.com/cockroachdb/pebble/objstorage/remote/storage.go

@@ -0,0 +1,126 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package remote
+
+import (
+	"context"
+	"io"
+)
+
+// Locator is an opaque string identifying a remote.Storage implementation.
+//
+// The Locator must not contain secrets (like authentication keys). Locators are
+// stored on disk in the shared object catalog and are passed around as part of
+// RemoteObjectBacking; they can also appear in error messages.
+type Locator string
+
+// StorageFactory is used to return Storage implementations based on locators. A
+// Pebble store that uses shared storage is configured with a StorageFactory.
+type StorageFactory interface {
+	CreateStorage(locator Locator) (Storage, error)
+}
+
+// SharedLevelsStart denotes the highest (i.e. lowest numbered) level that will
+// have sstables shared across Pebble instances when doing skip-shared
+// iteration (see db.ScanInternal) or shared file ingestion (see
+// db.IngestAndExcise).
+const SharedLevelsStart = 5
+
+// CreateOnSharedStrategy specifies what table files should be created on shared
+// storage. For use with CreateOnShared in options.
+type CreateOnSharedStrategy int
+
+const (
+	// CreateOnSharedNone denotes no files being created on shared storage.
+	CreateOnSharedNone CreateOnSharedStrategy = iota
+	// CreateOnSharedLower denotes the creation of files in lower levels of the
+	// LSM (specifically, L5 and L6 as they're below SharedLevelsStart) on
+	// shared storage, and higher levels on local storage.
+	CreateOnSharedLower
+	// CreateOnSharedAll denotes the creation of all sstables on shared storage.
+	CreateOnSharedAll
+)
+
+// ShouldCreateShared returns whether new table files at the specified level
+// should be created on shared storage.
+func ShouldCreateShared(strategy CreateOnSharedStrategy, level int) bool {
+	switch strategy {
+	case CreateOnSharedAll:
+		return true
+	case CreateOnSharedNone:
+		return false
+	case CreateOnSharedLower:
+		return level >= SharedLevelsStart
+	default:
+		panic("unexpected CreateOnSharedStrategy value")
+	}
+}
+
+// Storage is an interface for a blob storage driver. This is lower-level
+// than an FS-like interface, however FS/File-like abstractions can be built on
+// top of these methods.
+//
+// TODO(bilal): Consider pushing shared file obsoletion as well as path
+// generation behind this interface.
+type Storage interface {
+	io.Closer
+
+	// ReadObject returns an ObjectReader that can be used to perform reads on an
+	// object, along with the total size of the object.
+	ReadObject(ctx context.Context, objName string) (_ ObjectReader, objSize int64, _ error)
+
+	// CreateObject returns a writer for the object at the request name. A new
+	// empty object is created if CreateObject is called on an existing object.
+	//
+	// A Writer *must* be closed via either Close, and if closing returns a
+	// non-nil error, that error should be handled or reported to the user -- an
+	// implementation may buffer written data until Close and only then return
+	// an error, or Write may return an opaque io.EOF with the underlying cause
+	// returned by the subsequent Close().
+	//
+	// TODO(radu): if we encounter some unrelated error while writing to the
+	// WriteCloser, we'd want to abort the whole thing rather than letting Close
+	// finalize the upload.
+	CreateObject(objName string) (io.WriteCloser, error)
+
+	// List enumerates files within the supplied prefix, returning a list of
+	// objects within that prefix. If delimiter is non-empty, names which have the
+	// same prefix, prior to the delimiter but after the prefix, are grouped into a
+	// single result which is that prefix. The order that results are returned is
+	// undefined. If a prefix is specified, the prefix is trimmed from the result
+	// list.
+	//
+	// An example would be, if the storage contains objects a, b/4, b/5 and b/6,
+	// these would be the return values:
+	//   List("", "") -> ["a", "b/4", "b/5", "b/6"]
+	//   List("", "/") -> ["a", "b"]
+	//   List("b", "/") -> ["4", "5", "6"]
+	//   List("b", "") -> ["/4", "/5", "/6"]
+	List(prefix, delimiter string) ([]string, error)
+
+	// Delete removes the named object from the store.
+	Delete(objName string) error
+
+	// Size returns the length of the named object in bytesWritten.
+	Size(objName string) (int64, error)
+
+	// IsNotExistError returns true if the given error (returned by a method in
+	// this interface) indicates that the object does not exist.
+	IsNotExistError(err error) bool
+}
+
+// ObjectReader is used to perform reads on an object.
+type ObjectReader interface {
+	// ReadAt reads len(p) bytes into p starting at offset off.
+	//
+	// Does not return partial results; if offset + len(p) is past the end of the
+	// object, an error is returned.
+	//
+	// Clients of ReadAt can execute parallel ReadAt calls on the same
+	// ObjectReader.
+	ReadAt(ctx context.Context, p []byte, offset int64) error
+
+	Close() error
+}