[WIP] lint a bit, update deps

.golangci.yml

@@ -0,0 +1,67 @@
+run:
+  tests: false
+  skip-dirs:
+    - data
+    - vendor
+  modules-download-mode: readonly
+  allow-parallel-runners: true
+  go: '1.21'
+output:
+  sort-results: true
+linters-settings:
+  gosec:
+    excludes:
+      # Оно не используется для повышения безопасности или каких-то таких секъюрных вещей, оно используется для выборки
+      # произвольного элемента из массива, так что ИДИ НАХУЙ с этой проверкой и не еби мозги липовой безопасностью.
+      - G404 # Insecure random number source (rand)
+linters:
+  disable-all: true
+  enable:
+    - errcheck
+    - gosimple
+    - govet
+    - ineffassign
+    - staticcheck
+    - unused
+    - asasalint
+    - asciicheck
+    - bodyclose
+    - containedctx
+    - contextcheck
+    - decorder
+    - durationcheck
+    - errchkjson
+    - errorlint
+    - exhaustive
+    - exportloopref
+    - gci
+    - ginkgolinter
+    - gocheckcompilerdirectives
+    - gocritic
+    - godot
+    - gofmt
+    - goimports
+    - gosec
+    - gosmopolitan
+    - grouper
+    - importas
+    - ireturn
+    - makezero
+    - mirror
+    - nakedret
+    - nilerr
+    - nilnil
+    - nlreturn
+    - noctx
+    - nolintlint
+    - nonamedreturns
+    - predeclared
+    - promlinter
+    - reassign
+    - revive
+    - rowserrcheck
+    - stylecheck
+    - unconvert
+    - wastedassign
+    - whitespace
+    - wsl

config.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"os"
 
@@ -21,7 +20,7 @@ func parseConfig(path string) (myConfig, error) {
 
 	// Конфиг-файл длинноват для конфига, попробуем следующего кандидата.
 	if fileInfo.Size() > 65535 {
-		err := errors.New(fmt.Sprintf("Config file %s is too long for config, skipping", path))
+		err := fmt.Errorf("Config file %s is too long for config, skipping", path)
 
 		return myConfig{}, err
 	}
@@ -36,8 +35,11 @@ func parseConfig(path string) (myConfig, error) {
 	// Исходя из документации, hjson какбы умеет парсить "кривой" json, но парсит его в map-ку.
 	// Интереснее на выходе получить структурку: то есть мы вначале конфиг преобразуем в map-ку, затем эту map-ку
 	// сериализуем в json, а потом json преврщааем в стркутурку. Не очень эффективно, но он и не часто требуется.
-	var sampleConfig myConfig
+	var (
-	var tmp map[string]interface{}
+		sampleConfig myConfig
+		tmp          map[string]interface{}
+	)
+
 	err = hjson.Unmarshal(buf, &tmp)
 
 	// Не удалось распарсить.
@@ -59,6 +61,7 @@ func parseConfig(path string) (myConfig, error) {
 	// Валидируем значения из конфига.
 	if sampleConfig.Redis.Server == "" {
 		sampleConfig.Redis.Server = "localhost"
+
 		log.Infof("Redis server is not defined in config, using localhost")
 	}
 
@@ -67,19 +70,19 @@ func parseConfig(path string) (myConfig, error) {
 	}
 
 	if sampleConfig.Redis.Channel == "" {
-		err := errors.New(fmt.Sprintf("Channel field in config file %s must be set", path))
+		err := fmt.Errorf("Channel field in config file %s must be set", path)
 
 		return myConfig{}, err
 	}
 
 	if sampleConfig.Redis.MyChannel == "" {
-		err := errors.New(fmt.Sprintf("My_channel field in config file %s must be set", path))
+		err := fmt.Errorf("My_channel field in config file %s must be set", path)
 
 		return myConfig{}, err
 	}
 
 	if sampleConfig.Telegram.Token == "" {
-		return myConfig{}, errors.New(fmt.Sprintf("telegram.token field in config file %s must be set", path))
+		return myConfig{}, fmt.Errorf("telegram.token field in config file %s must be set", path)
 	}
 
 	if sampleConfig.Loglevel == "" {
@@ -89,7 +92,7 @@ func parseConfig(path string) (myConfig, error) {
 	// sampleConfig.Log = "" if not set
 
 	if sampleConfig.Csign == "" {
-		err := errors.New(fmt.Sprintf("Csign field in config file %s must be set", path))
+		err := fmt.Errorf("Csign field in config file %s must be set", path)
 
 		return myConfig{}, err
 	}
@@ -99,7 +102,7 @@ func parseConfig(path string) (myConfig, error) {
 	}
 
 	if sampleConfig.DataDir == "" {
-		return myConfig{}, errors.New(fmt.Sprintf("Data_dir field in config file %s must be set", path))
+		return myConfig{}, fmt.Errorf("Data_dir field in config file %s must be set", path)
 	}
 
 	return sampleConfig, err

globals.go

@@ -9,13 +9,13 @@ import (
 	"github.com/go-redis/redis/v8"
 )
 
-// Config - это у нас глобальная штука :)
+// Config - это у нас глобальная штука.
 var config myConfig
 
-// To break circular message forwarding we must set some sane default, it can be overridden via config
+// To break circular message forwarding we must set some sane default, it can be overridden via config.
 var forwardMax int64 = 5
 
-// Объектики клиента-редиски
+// Объектики клиента-редиски.
 var redisClient *redis.Client
 var subscriber *redis.PubSub
 
@@ -25,10 +25,10 @@ var ctx = context.Background()
 // Ставится в true, если мы получили сигнал на выключение
 var shutdown = false
 
-// Канал, в который приходят уведомления для хэндлера сигналов от траппера сигналов
+// Канал, в который приходят уведомления для хэндлера сигналов от траппера сигналов.
 var sigChan = make(chan os.Signal, 1)
 
-// Мапка с открытыми дескрипторами баз с настройками
+// Мапка с открытыми дескрипторами баз с настройками.
 var settingsDB = make(map[string]*pebble.DB)
 
 var tg echotron.API

go.mod

@@ -3,9 +3,9 @@ module aleesa-telegram-go
 go 1.19
 
 require (
-	github.com/NicoNex/echotron/v3 v3.28.0
+	github.com/NicoNex/echotron/v3 v3.29.0
 	github.com/cockroachdb/errors v1.11.1
-	github.com/cockroachdb/pebble v1.0.0
+	github.com/cockroachdb/pebble v1.1.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/hjson/hjson-go v3.3.0+incompatible
@@ -18,21 +18,22 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/getsentry/sentry-go v0.26.0 // indirect
+	github.com/getsentry/sentry-go v0.27.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.18.0 // indirect
-	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/client_model v0.6.0 // indirect
-	github.com/prometheus/common v0.46.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
-	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
+	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
-	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/protobuf v1.32.0 // indirect
 )

go.sum

@@ -1,20 +1,22 @@
 github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
 github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
-github.com/NicoNex/echotron/v3 v3.28.0 h1:A5uGWlD1iQYJe41XYxmb/sAQ8ZoDSJgbB2+Ix9PDimY=
+github.com/NicoNex/echotron/v3 v3.29.0 h1:STzeYNauF+h82vTL9QIqIBEvhke/bsxgYKQMCmJKnLM=
-github.com/NicoNex/echotron/v3 v3.28.0/go.mod h1:LpP5IyHw0y+DZUZMBgXEDAF9O8feXrQu7w7nlJzzoZI=
+github.com/NicoNex/echotron/v3 v3.29.0/go.mod h1:LpP5IyHw0y+DZUZMBgXEDAF9O8feXrQu7w7nlJzzoZI=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877 h1:1MLK4YpFtIEo3ZtMA5C795Wtv5VuUnrXX7mQG+aHg6o=
+github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
 github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8=
 github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v1.0.0 h1:WZWlV/s78glZbY2ylUITDOWSVBD3cLjcWPLRPFbHNYg=
+github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4=
-github.com/cockroachdb/pebble v1.0.0/go.mod h1:bynZ3gvVyhlvjLI7PT6dmZ7g76xzJ7HpxfjgkzCGz6s=
+github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E=
 github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -22,8 +24,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
-github.com/getsentry/sentry-go v0.26.0 h1:IX3++sF6/4B5JcevhdZfdKIHfyvMmAq/UnqcyT2H6mA=
+github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
-github.com/getsentry/sentry-go v0.26.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
+github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
@@ -31,13 +33,13 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/hjson/hjson-go v3.3.0+incompatible h1:Rqr+Ya+0aCJMjaE4s8E9YKvuJLuLVpEvz4ONum52vnI=
 github.com/hjson/hjson-go v3.3.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
-github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -53,10 +55,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
 github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
-github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos=
-github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8=
-github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
-github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
@@ -72,8 +74,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
+golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ=
-golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
+golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -89,8 +91,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=

msg-parsers.go

@@ -11,10 +11,10 @@ import (
 
 // telego msg parser
 
-// redisMsgParser парсит json-чики прилетевшие из REDIS-ки, причём, json-чики должны быть относительно валидными
+// redisMsgParser парсит json-чики прилетевшие из REDIS-ки, причём, json-чики должны быть относительно валидными.
 func redisMsgParser(msg string) {
 	if shutdown {
-		// Если мы завершаем работу программы, то нам ничего обрабатывать не надо
+		// Если мы завершаем работу программы, то нам ничего обрабатывать не надо.
 		return
 	}
 
@@ -24,48 +24,56 @@ func redisMsgParser(msg string) {
 
 	if err := json.Unmarshal([]byte(msg), &j); err != nil {
 		log.Warnf("Unable to to parse message from redis channel: %s", err)
+
 		return
 	}
 
 	// Validate our j
 	if exist := j.From; exist == "" {
 		log.Warnf("Incorrect msg from redis, no from field: %s", msg)
+
 		return
 	}
 
 	if exist := j.Chatid; exist == "" {
 		log.Warnf("Incorrect msg from redis, no chatid field: %s", msg)
+
 		return
 	}
 
 	if exist := j.Userid; exist == "" {
 		log.Warnf("Incorrect msg from redis, no userid field: %s", msg)
+
 		return
 	}
 
 	if exist := j.Message; exist == "" {
 		log.Warnf("Incorrect msg from redis, no message field: %s", msg)
+
 		return
 	}
 
 	if exist := j.Plugin; exist == "" {
 		log.Warnf("Incorrect msg from redis, no plugin field: %s", msg)
+
 		return
 	}
 
 	if exist := j.Mode; exist == "" {
 		log.Warnf("Incorrect msg from redis, no mode field: %s", msg)
+
 		return
 	}
 
 	// j.Misc.Answer может и не быть, тогда ответа на такое сообщение не будет
 	if j.Misc.Answer == 0 {
 		log.Debug("Field Misc->Answer = 0, skipping message")
+
 		return
 	}
 
 	// j.Misc.BotNick тоже можно не передавать, тогда будет записана пустая строка
-	// j.Misc.CSign если нам его не передали, возьмём значение из конфига
+	// j.Misc.CSign если нам его не передали, возьмём значение из конфига. (но по идее нам оно тут не нужнО.)
 	if exist := j.Misc.Csign; exist == "" {
 		j.Misc.Csign = config.Csign
 	}
@@ -82,6 +90,7 @@ func redisMsgParser(msg string) {
 	// Отвалидировались, теперь вернёмся к нашим баранам.
 
 	var opts *echotron.MessageOptions
+
 	chatid, err := strconv.ParseInt(j.Chatid, 10, 64)
 
 	if err != nil {
@@ -100,8 +109,6 @@ func redisMsgParser(msg string) {
 		log.Errorf("Unable to send message to telegram api: %s", err)
 		log.Errorf("Response dump: %s", spew.Sdump(resp))
 	}
-
-	return
 }
 
 // telegramMsgParser парсит ивент, прилетевший из bot api.
@@ -128,11 +135,14 @@ func telegramMsgParser(msg *echotron.Update) {
 		// TODO: проверить, надо ли прощаться с ушедшим участником чата и попрощаться, если надо
 	}
 
+	var rmsg rMsg
+
 	switch msg.Message.Chat.Type {
 	case "private":
 		// handle private messages
-		// всегда ставим флажок, что надо отвечать.
+		rmsg.Misc.Answer = 1
 		// если фраза является командой - засылаем её в парсер команд
+
 		// Засылаем фразу в misc-канал (в роутер)
 	case "group", "supergroup":
 		// handle public (super)group messages
@@ -144,11 +154,14 @@ func telegramMsgParser(msg *echotron.Update) {
 		// Здесь же, пытаемся убрать из фразы ник или имя бота
 		// Засылаем фразу в misc-канал (в роутер)
 	case "channel":
-		// Тут мы ничего сделать не можем
+		// Тут мы ничего сделать не можем.
+	} //nolint:wsl
+}
 
-	}
+func cmdParser(cmd string) error {
+	var err error
 
-	return
+	return err
 }
 
 /* vim: set ft=go noet ai ts=4 sw=4 sts=4: */

settings-db-util.go

@@ -11,7 +11,7 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-// StoreKV сохраняет в указанной бд ключ и значение
+// StoreKV сохраняет в указанной бд ключ и значение.
 func StoreKV(db *pebble.DB, key string, value string) error {
 	var kArray = []byte(key)
 	var vArray = []byte(value)
@@ -40,7 +40,7 @@ func FetchV(db *pebble.DB, key string) (string, error) {
 	return valueString, err
 }
 
-// Достанем настройку из БД с настройками
+// getSetting достаёт настройку из БД с настройками.
 func getSetting(chatID string, setting string) string {
 	var err error
 
@@ -82,7 +82,7 @@ func getSetting(chatID string, setting string) string {
 	return value
 }
 
-// Сохраним настройку в БД с настройками
+// saveSetting сохраняет настройку в БД с настройками
 func saveSetting(chatID string, setting string, value string) error {
 	var chatHash = sha256.Sum256([]byte(chatID))
 	var database = fmt.Sprintf("settings_db/%x", chatHash)

types.go

@@ -1,6 +1,6 @@
 package main
 
-// Конфиг
+// myConfig структура, описывающая формат Конфига.
 type myConfig struct {
 	Redis struct {
 		Server    string `json:"server,omitempty"`
@@ -20,7 +20,7 @@ type myConfig struct {
 	DataDir     string `json:"data_dir,omitempty"`
 }
 
-// Входящее сообщение из pubsub-канала redis-ки
+// rMsg структура, описывающая формат входящего сообщения из pubsub-канала redis-ки.
 type rMsg struct {
 	From    string `json:"from,omitempty"`
 	Chatid  string `json:"chatid,omitempty"`
@@ -39,7 +39,7 @@ type rMsg struct {
 	} `json:"Misc"`
 }
 
-// Исходящее сообщение в pubsub-канал redis-ки
+// sMsg структура, описывающая исходящее сообщение в pubsub-канал redis-ки.
 type sMsg struct {
 	From    string `json:"from"`
 	Chatid  string `json:"chatid"`

util.go

@@ -3,7 +3,6 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"syscall"
@@ -12,6 +11,7 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+// readConfig считывает и парсит конфиг-файл.
 func readConfig() {
 	configLoaded := false
 	executablePath, err := os.Executable()
@@ -40,27 +40,33 @@ func readConfig() {
 		// Конфиг-файл длинноват для конфига, попробуем следующего кандидата
 		if fileInfo.Size() > 65535 {
 			log.Warnf("Config file %s is too long for config, skipping", location)
+
 			continue
 		}
 
-		buf, err := ioutil.ReadFile(location)
+		buf, err := os.ReadFile(location)
 
 		// Не удалось прочитать, попробуем следующего кандидата
 		if err != nil {
 			log.Warnf("Skip reading config file %s: %s", location, err)
+
 			continue
 		}
 
 		// Исходя из документации, hjson какбы умеет парсить "кривой" json, но парсит его в map-ку.
 		// Интереснее на выходе получить структурку: то есть мы вначале конфиг преобразуем в map-ку, затем эту map-ку
 		// сериализуем в json, а потом json преврщааем в стркутурку. Не очень эффективно, но он и не часто требуется.
-		var sampleConfig myConfig
+		var (
-		var tmp map[string]interface{}
+			sampleConfig myConfig
+			tmp          map[string]interface{}
+		)
+
 		err = hjson.Unmarshal(buf, &tmp)
 
 		// Не удалось распарсить - попробуем следующего кандидата
 		if err != nil {
 			log.Warnf("Skip parsing config file %s: %s", location, err)
+
 			continue
 		}
 
@@ -69,17 +75,20 @@ func readConfig() {
 		// Не удалось преобразовать map-ку в json
 		if err != nil {
 			log.Warnf("Skip parsing config file %s: %s", location, err)
+
 			continue
 		}
 
 		if err := json.Unmarshal(tmpjson, &sampleConfig); err != nil {
 			log.Warnf("Skip parsing config file %s: %s", location, err)
+
 			continue
 		}
 
-		// Валидируем значения из конфига
+		// Валидируем значения из конфига.
 		if sampleConfig.Redis.Server == "" {
 			sampleConfig.Redis.Server = "localhost"
+
 			log.Infof("Redis server is not defined in config, using localhost")
 		}
 
@@ -119,7 +128,9 @@ func readConfig() {
 
 		config = sampleConfig
 		configLoaded = true
+
 		log.Infof("Using %s as config file", location)
+
 		break
 	}
 
@@ -129,7 +140,7 @@ func readConfig() {
 	}
 }
 
-// Хэндлер сигналов закрывает все бд, все сетевые соединения и сваливает из приложения
+// sigHandler хэндлер сигналов закрывает все бд, все сетевые соединения и сваливает из приложения.
 func sigHandler() {
 	var err error
 
@@ -148,10 +159,10 @@ func sigHandler() {
 			continue
 		}
 
-		// Чтобы не срать в логи ошибками от редиски, проставим shutdown state приложения в true
+		// Чтобы не срать в логи ошибками от редиски, проставим shutdown state приложения в true.
 		shutdown = true
 
-		// Отпишемся от всех каналов и закроем коннект к редиске
+		// Отпишемся от всех каналов и закроем коннект к редиске.
 		if err = subscriber.Unsubscribe(ctx); err != nil {
 			log.Errorf("Unable to unsubscribe from redis channels cleanly: %s", err)
 		} else {
@@ -164,7 +175,7 @@ func sigHandler() {
 			log.Debug("Close redis connection")
 		}
 
-		// close all telego stuff
+		// close all telego stuff.
 
 		if len(settingsDB) > 0 {
 			log.Debug("Closing runtime irc channel settings db")

vendor/github.com/NicoNex/echotron/v3/types.go

@@ -489,11 +489,13 @@ type Chat struct {
 	InviteLink                         string           `json:"invite_link,omitempty"`
 	EmojiStatusCustomEmojiID           string           `json:"emoji_status_custom_emoji_id,omitempty"`
 	Type                               string           `json:"type"`
+	CustomEmojiStickerSetName          string           `json:"custom_emoji_sticker_set_name,omitempty"`
 	AccentColorID                      int              `json:"accent_color_id,omitempty"`
 	ProfileAccentColorID               int              `json:"profile_accent_color_id,omitempty"`
 	EmojiStatusExpirationDate          int              `json:"emoji_status_expiration_date,omitempty"`
 	MessageAutoDeleteTime              int              `json:"message_auto_delete_time,omitempty"`
 	SlowModeDelay                      int              `json:"slow_mode_delay,omitempty"`
+	UnrestrictBoostCount               int              `json:"unrestrict_boost_count,omitempty"`
 	LinkedChatID                       int64            `json:"linked_chat_id,omitempty"`
 	ID                                 int64            `json:"id"`
 	IsForum                            bool             `json:"is_forum,omitempty"`
@@ -553,10 +555,12 @@ type Message struct {
 	UsersShared                   *UsersShared                   `json:"users_shared,omitempty"`
 	ChatShared                    *ChatShared                    `json:"chat_shared,omitempty"`
 	Story                         *Story                         `json:"story,omitempty"`
+	ReplyToStory                  *Story                         `json:"reply_to_story,omitempty"`
 	ExternalReply                 *ExternalReplyInfo             `json:"external_reply,omitempty"`
 	Quote                         *TextQuote                     `json:"quote,omitempty"`
 	LinkPreviewOptions            *LinkPreviewOptions            `json:"link_preview_options,omitempty"`
 	ForwardOrigin                 *MessageOrigin                 `json:"forward_origin,omitempty"`
+	BoostAdded                    *ChatBoostAdded                `json:"boost_added,omitempty"`
 	MediaGroupID                  string                         `json:"media_group_id,omitempty"`
 	ConnectedWebsite              string                         `json:"connected_website,omitempty"`
 	NewChatTitle                  string                         `json:"new_chat_title,omitempty"`
@@ -575,6 +579,7 @@ type Message struct {
 	Date                          int                            `json:"date"`
 	MigrateToChatID               int                            `json:"migrate_to_chat_id,omitempty"`
 	EditDate                      int                            `json:"edit_date,omitempty"`
+	SenderBoostCount              int                            `json:"sender_boost_count,omitempty"`
 	DeleteChatPhoto               bool                           `json:"delete_chat_photo,omitempty"`
 	IsTopicMessage                bool                           `json:"is_topic_message,omitempty"`
 	IsAutomaticForward            bool                           `json:"is_automatic_forward,omitempty"`
@@ -1173,6 +1178,11 @@ type ChatJoinRequest struct {
 	UserChatID int64           `json:"user_chat_id"`
 }
 
+// ChatBoostAdded represents a service message about a user boosting a chat.
+type ChatBoostAdded struct {
+	BoostCount int `json:"boost_count"`
+}
+
 // ForumTopicCreated represents a service message about a new forum topic created in the chat.
 type ForumTopicCreated struct {
 	Name              string `json:"name"`
@@ -1238,9 +1248,11 @@ type ChatShared struct {
 	ChatID    int64 `json:"chat_id"`
 }
 
-// Story represents a message about a forwarded story in the chat.
+// Story represents a story.
-// Currently holds no information.
+type Story struct {
-type Story struct{}
+	Chat Chat  `json:"chat"`
+	ID   int64 `json:"id"`
+}
 
 type ReactionType interface {
 	ImplementsReactionType()

vendor/github.com/cockroachdb/pebble/.gitignore

@@ -1,6 +1,9 @@
+# Github action artifacts.
+artifacts
 # Profiling artifacts.
 cpu.*.prof
 heap.prof
 mutex.prof
+coverprofile.out
 # Testing artifacts
 meta.*.test

vendor/github.com/cockroachdb/pebble/Makefile

@@ -4,7 +4,8 @@ GOFLAGS :=
 STRESSFLAGS :=
 TAGS := invariants
 TESTS := .
-LATEST_RELEASE := $(shell git fetch origin && git branch -r --list '*/crl-release-*' | grep -o 'crl-release-.*$$' | sort | tail -1)
+PREV_RELEASE := crl-release-23.1
+COVER_PROFILE := coverprofile.out
 
 .PHONY: all
 all:
@@ -15,7 +16,10 @@ all:
 	@echo "  make stressrace"
 	@echo "  make stressmeta"
 	@echo "  make crossversion-meta"
+	@echo "  make testcoverage"
 	@echo "  make mod-update"
+	@echo "  make generate"
+	@echo "  make generate-test-data"
 	@echo "  make clean"
 
 override testflags :=
@@ -23,6 +27,10 @@ override testflags :=
 test:
 	${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG}
 
+.PHONY: testcoverage
+testcoverage:
+	${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG} -coverprofile ${COVER_PROFILE}
+
 .PHONY: testrace
 testrace: testflags += -race -timeout 20m
 testrace: test
@@ -34,6 +42,10 @@ testmsan: export CC=clang
 testmsan: testflags += -msan -timeout 20m
 testmsan: test
 
+.PHONY: testobjiotracing
+testobjiotracing:
+	${GO} test -tags '$(TAGS) pebble_obj_io_tracing' ${testflags} -run ${TESTS} ./objstorage/objstorageprovider/objiotracing
+
 .PHONY: lint
 lint:
 	${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ./internal/lint
@@ -51,20 +63,31 @@ stressmeta: stress
 
 .PHONY: crossversion-meta
 crossversion-meta:
-	git checkout ${LATEST_RELEASE}; \
+	git checkout ${PREV_RELEASE}; \
-		${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${LATEST_RELEASE}.test'; \
+		${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${PREV_RELEASE}.test'; \
 		git checkout -; \
 		${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/head.test'; \
-		${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${LATEST_RELEASE},${LATEST_RELEASE},${LATEST_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
+		${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${PREV_RELEASE},${PREV_RELEASE},${PREV_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
 
 .PHONY: stress-crossversion
 stress-crossversion:
-	STRESS=1 ./scripts/run-crossversion-meta.sh crl-release-21.2 crl-release-22.1 crl-release-22.2 master
+	STRESS=1 ./scripts/run-crossversion-meta.sh crl-release-21.2 crl-release-22.1 crl-release-22.2 crl-release-23.1 master
 
 .PHONY: generate
 generate:
 	${GO} generate ${PKG}
 
+generate:
+
+# Note that the output of generate-test-data is not deterministic. This should
+# only be run manually as needed.
+.PHONY: generate-test-data
+generate-test-data:
+	${GO} run -tags make_incorrect_manifests ./tool/make_incorrect_manifests.go
+	${GO} run -tags make_test_find_db ./tool/make_test_find_db.go
+	${GO} run -tags make_test_sstables ./tool/make_test_sstables.go
+	${GO} run -tags make_test_remotecat ./tool/make_test_remotecat.go
+
 mod-update:
 	${GO} get -u
 	${GO} mod tidy
@@ -92,9 +115,11 @@ endif
 	@${GO} mod tidy
 	$(MAKE) git-clean-check
 
+# TODO(radu): switch back to @latest once bogus doc changes are
+# addressed; see https://github.com/cockroachdb/crlfmt/pull/44
 .PHONY: format
 format:
-	go install github.com/cockroachdb/crlfmt@latest && crlfmt -w -tab 2 .
+	go install github.com/cockroachdb/crlfmt@44a36ec7 && crlfmt -w -tab 2 .
 
 .PHONY: format-check
 format-check:

vendor/github.com/cockroachdb/pebble/README.md

@@ -1,4 +1,4 @@
-# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble)
+# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble) <sup><sub><sub>[Coverage](https://storage.googleapis.com/crl-codecover-public/pebble/index.html)</sub></sub></sup>
 
 #### [Nightly benchmarks](https://cockroachdb.github.io/pebble/)
 
@@ -8,9 +8,8 @@ RocksDB file formats and a few extensions such as range deletion
 tombstones, table-level bloom filters, and updates to the MANIFEST
 format.
 
-Pebble intentionally does not aspire to include every feature in
+Pebble intentionally does not aspire to include every feature in RocksDB and
-RocksDB and is specifically targetting the use case and feature set
+specifically targets the use case and feature set needed by CockroachDB:
-needed by CockroachDB:
 
 * Block-based tables
 * Checkpoints
@@ -68,6 +67,17 @@ Pebble offers several improvements over RocksDB:
 * Faster commit pipeline that achieves better concurrency.
 * Seamless merged iteration of indexed batches. The mutations in the
   batch conceptually occupy another memtable level.
+* L0 sublevels and flush splitting for concurrent compactions out of L0 and
+  reduced read-amplification during heavy write load.
+* Faster LSM edits in LSMs with large numbers of sstables through use of a
+  copy-on-write B-tree to hold file metadata.
+* Delete-only compactions that drop whole sstables that fall within the bounds
+  of a range deletion.
+* Block-property collectors and filters that enable iterators to skip tables,
+  index blocks and data blocks that are irrelevant, according to user-defined
+  properties over key-value pairs.
+* Range keys API, allowing KV pairs defined over a range of keyspace with
+  user-defined semantics and interleaved during iteration.
 * Smaller, more approachable code base.
 
 See the [Pebble vs RocksDB: Implementation
@@ -100,10 +110,68 @@ known incompatibilities.
   format.
 * Plain table format. Pebble does not support the plain table sstable
   format.
-* SSTable format version 3 and 4. Pebble does not currently support
+* SSTable format version 3 and 4. Pebble does not support version 3
-  version 3 and version 4 format sstables. The sstable format version
+  and version 4 format sstables. The sstable format version is
-  is controlled by the `BlockBasedTableOptions::format_version`
+  controlled by the `BlockBasedTableOptions::format_version` option.
-  option. See [#97](https://github.com/cockroachdb/pebble/issues/97).
+  See [#97](https://github.com/cockroachdb/pebble/issues/97).
+
+## Format major versions
+
+Over time Pebble has introduced new physical file formats.  Backwards
+incompatible changes are made through the introduction of 'format major
+versions'. By default, when Pebble opens a database, it defaults to
+`FormatMostCompatible`. This version is bi-directionally compatible with RocksDB
+6.2.1 (with the caveats described above).
+
+To opt into new formats, a user may set `FormatMajorVersion` on the
+[`Options`](https://pkg.go.dev/github.com/cockroachdb/pebble#Options)
+supplied to
+[`Open`](https://pkg.go.dev/github.com/cockroachdb/pebble#Open), or
+upgrade the format major version at runtime using
+[`DB.RatchetFormatMajorVersion`](https://pkg.go.dev/github.com/cockroachdb/pebble#DB.RatchetFormatMajorVersion).
+Format major version upgrades are permanent; There is no option to
+return to an earlier format.
+
+The table below outlines the history of format major versions:
+
+| Name                               | Value | Migration  |
+|------------------------------------|-------|------------|
+| FormatMostCompatible               |   1   | No         |
+| FormatVersioned                    |   3   | No         |
+| FormatSetWithDelete                |   4   | No         |
+| FormatBlockPropertyCollector       |   5   | No         |
+| FormatSplitUserKeysMarked          |   6   | Background |
+| FormatSplitUserKeysMarkedCompacted |   7   | Blocking   |
+| FormatRangeKeys                    |   8   | No         |
+| FormatMinTableFormatPebblev1       |   9   | No         |
+| FormatPrePebblev1Marked            |  10   | Background |
+| FormatSSTableValueBlocks           |  12   | No         |
+| FormatFlushableIngest              |  13   | No         |
+| FormatPrePebblev1MarkedCompacted   |  14   | Blocking   |
+| FormatDeleteSizedAndObsolete       |  15   | No         |
+| FormatVirtualSSTables              |  16   | No         |
+
+Upgrading to a format major version with 'Background' in the migration
+column may trigger background activity to rewrite physical file
+formats, typically through compactions. Upgrading to a format major
+version with 'Blocking' in the migration column will block until a
+migration is complete. The database may continue to serve reads and
+writes if upgrading a live database through
+`RatchetFormatMajorVersion`, but the method call will not return until
+the migration is complete.
+
+For reference, the table below lists the range of supported Pebble format major
+versions for CockroachDB releases.
+
+| CockroachDB release | Earliest supported                 | Latest supported          |
+|---------------------|------------------------------------|---------------------------|
+| 20.1 through 21.1   | FormatMostCompatible               | FormatMostCompatible      |
+| 21.2                | FormatMostCompatible               | FormatSetWithDelete       |
+| 21.2                | FormatMostCompatible               | FormatSetWithDelete       |
+| 22.1                | FormatMostCompatible               | FormatSplitUserKeysMarked |
+| 22.2                | FormatMostCompatible               | FormatPrePebblev1Marked   |
+| 23.1                | FormatSplitUserKeysMarkedCompacted | FormatFlushableIngest     |
+| 23.2                | FormatSplitUserKeysMarkedCompacted | FormatVirtualSSTables     |
 
 ## Pedigree
 

vendor/github.com/cockroachdb/pebble/batch.go

@@ -41,10 +41,10 @@ const (
 var ErrNotIndexed = errors.New("pebble: batch not indexed")
 
 // ErrInvalidBatch indicates that a batch is invalid or otherwise corrupted.
-var ErrInvalidBatch = errors.New("pebble: invalid batch")
+var ErrInvalidBatch = base.MarkCorruptionError(errors.New("pebble: invalid batch"))
 
 // ErrBatchTooLarge indicates that a batch is invalid or otherwise corrupted.
-var ErrBatchTooLarge = errors.Newf("pebble: batch too large: >= %s", humanize.Uint64(maxBatchSize))
+var ErrBatchTooLarge = base.MarkCorruptionError(errors.Newf("pebble: batch too large: >= %s", humanize.Bytes.Uint64(maxBatchSize)))
 
 // DeferredBatchOp represents a batch operation (eg. set, merge, delete) that is
 // being inserted into the batch. Indexing is not performed on the specified key
@@ -183,6 +183,13 @@ func (d DeferredBatchOp) Finish() error {
 // WAL, and thus stable. New record kinds may be added, but the existing ones
 // will not be modified.
 type Batch struct {
+	batchInternal
+	applied atomic.Bool
+}
+
+// batchInternal contains the set of fields within Batch that are non-atomic and
+// capable of being reset using a *b = batchInternal{} struct copy.
+type batchInternal struct {
 	// Data is the wire format of a batch's log entry:
 	//   - 8 bytes for a sequence number of the first batch element,
 	//     or zeroes if the batch has not yet been applied,
@@ -260,10 +267,12 @@ type Batch struct {
 	// memtable.
 	flushable *flushableBatch
 
-	// ingestedSSTBatch indicates that the batch contains one or more key kinds
+	// minimumFormatMajorVersion indicates the format major version required in
-	// of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST
+	// order to commit this batch. If an operation requires a particular format
-	// then it will only contain key kinds of IngestSST.
+	// major version, it ratchets the batch's minimumFormatMajorVersion. When
-	ingestedSSTBatch bool
+	// the batch is committed, this is validated against the database's current
+	// format major version.
+	minimumFormatMajorVersion FormatMajorVersion
 
 	// Synchronous Apply uses the commit WaitGroup for both publishing the
 	// seqnum and waiting for the WAL fsync (if needed). Asynchronous
@@ -285,7 +294,22 @@ type Batch struct {
 	commitStats BatchCommitStats
 
 	commitErr error
-	applied   uint32 // updated atomically
+
+	// Position bools together to reduce the sizeof the struct.
+
+	// ingestedSSTBatch indicates that the batch contains one or more key kinds
+	// of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST
+	// then it will only contain key kinds of IngestSST.
+	ingestedSSTBatch bool
+
+	// committing is set to true when a batch begins to commit. It's used to
+	// ensure the batch is not mutated concurrently. It is not an atomic
+	// deliberately, so as to avoid the overhead on batch mutations. This is
+	// okay, because under correct usage this field will never be accessed
+	// concurrently. It's only under incorrect usage the memory accesses of this
+	// variable may violate memory safety. Since we don't use atomics here,
+	// false negatives are possible.
+	committing bool
 }
 
 // BatchCommitStats exposes stats related to committing a batch.
@@ -312,7 +336,11 @@ type BatchCommitStats struct {
 	// commitPipeline.Commit.
 	SemaphoreWaitDuration time.Duration
 	// WALQueueWaitDuration is the wait time for allocating memory blocks in the
-	// LogWriter (due to the LogWriter not writing fast enough).
+	// LogWriter (due to the LogWriter not writing fast enough). At the moment
+	// this is duration is always zero because a single WAL will allow
+	// allocating memory blocks up to the entire memtable size. In the future,
+	// we may pipeline WALs and bound the WAL queued blocks separately, so this
+	// field is preserved for that possibility.
 	WALQueueWaitDuration time.Duration
 	// MemTableWriteStallDuration is the wait caused by a write stall due to too
 	// many memtables (due to not flushing fast enough).
@@ -356,6 +384,14 @@ func newBatch(db *DB) *Batch {
 	return b
 }
 
+func newBatchWithSize(db *DB, size int) *Batch {
+	b := newBatch(db)
+	if cap(b.data) < size {
+		b.data = rawalloc.New(0, size)
+	}
+	return b
+}
+
 func newIndexedBatch(db *DB, comparer *Comparer) *Batch {
 	i := indexedBatchPool.Get().(*indexedBatch)
 	i.batch.cmp = comparer.Compare
@@ -367,6 +403,14 @@ func newIndexedBatch(db *DB, comparer *Comparer) *Batch {
 	return &i.batch
 }
 
+func newIndexedBatchWithSize(db *DB, comparer *Comparer, size int) *Batch {
+	b := newIndexedBatch(db, comparer)
+	if cap(b.data) < size {
+		b.data = rawalloc.New(0, size)
+	}
+	return b
+}
+
 // nextSeqNum returns the batch "sequence number" that will be given to the next
 // key written to the batch. During iteration keys within an indexed batch are
 // given a sequence number consisting of their offset within the batch combined
@@ -405,17 +449,21 @@ func (b *Batch) release() {
 	}
 }
 
-func (b *Batch) refreshMemTableSize() {
+func (b *Batch) refreshMemTableSize() error {
 	b.memTableSize = 0
 	if len(b.data) < batchHeaderLen {
-		return
+		return nil
 	}
 
 	b.countRangeDels = 0
 	b.countRangeKeys = 0
+	b.minimumFormatMajorVersion = 0
 	for r := b.Reader(); ; {
-		kind, key, value, ok := r.Next()
+		kind, key, value, ok, err := r.Next()
 		if !ok {
+			if err != nil {
+				return err
+			}
 			break
 		}
 		switch kind {
@@ -423,12 +471,23 @@ func (b *Batch) refreshMemTableSize() {
 			b.countRangeDels++
 		case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
 			b.countRangeKeys++
+		case InternalKeyKindDeleteSized:
+			if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete {
+				b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete
+			}
 		case InternalKeyKindIngestSST:
+			if b.minimumFormatMajorVersion < FormatFlushableIngest {
+				b.minimumFormatMajorVersion = FormatFlushableIngest
+			}
 			// This key kind doesn't contribute to the memtable size.
 			continue
 		}
 		b.memTableSize += memTableEntrySize(len(key), len(value))
 	}
+	if b.countRangeKeys > 0 && b.minimumFormatMajorVersion < FormatRangeKeys {
+		b.minimumFormatMajorVersion = FormatRangeKeys
+	}
+	return nil
 }
 
 // Apply the operations contained in the batch to the receiver batch.
@@ -442,7 +501,7 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error {
 		return nil
 	}
 	if len(batch.data) < batchHeaderLen {
-		return base.CorruptionErrorf("pebble: invalid batch")
+		return ErrInvalidBatch
 	}
 
 	offset := len(b.data)
@@ -459,8 +518,11 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error {
 		// order to update the index.
 		for iter := BatchReader(b.data[offset:]); len(iter) > 0; {
 			offset := uintptr(unsafe.Pointer(&iter[0])) - uintptr(unsafe.Pointer(&b.data[0]))
-			kind, key, value, ok := iter.Next()
+			kind, key, value, ok, err := iter.Next()
 			if !ok {
+				if err != nil {
+					return err
+				}
 				break
 			}
 			switch kind {
@@ -516,6 +578,9 @@ func (b *Batch) Get(key []byte) ([]byte, io.Closer, error) {
 }
 
 func (b *Batch) prepareDeferredKeyValueRecord(keyLen, valueLen int, kind InternalKeyKind) {
+	if b.committing {
+		panic("pebble: batch already committing")
+	}
 	if len(b.data) == 0 {
 		b.init(keyLen + valueLen + 2*binary.MaxVarintLen64 + batchHeaderLen)
 	}
@@ -565,6 +630,9 @@ func (b *Batch) prepareDeferredKeyValueRecord(keyLen, valueLen int, kind Interna
 }
 
 func (b *Batch) prepareDeferredKeyRecord(keyLen int, kind InternalKeyKind) {
+	if b.committing {
+		panic("pebble: batch already committing")
+	}
 	if len(b.data) == 0 {
 		b.init(keyLen + binary.MaxVarintLen64 + batchHeaderLen)
 	}
@@ -598,6 +666,41 @@ func (b *Batch) prepareDeferredKeyRecord(keyLen int, kind InternalKeyKind) {
 	b.data = b.data[:pos+keyLen]
 }
 
+// AddInternalKey allows the caller to add an internal key of point key kinds to
+// a batch. Passing in an internal key of kind RangeKey* or RangeDelete will
+// result in a panic. Note that the seqnum in the internal key is effectively
+// ignored, even though the Kind is preserved. This is because the batch format
+// does not allow for a per-key seqnum to be specified, only a batch-wide one.
+//
+// Note that non-indexed keys (IngestKeyKind{LogData,IngestSST}) are not
+// supported with this method as they require specialized logic.
+func (b *Batch) AddInternalKey(key *base.InternalKey, value []byte, _ *WriteOptions) error {
+	keyLen := len(key.UserKey)
+	hasValue := false
+	switch key.Kind() {
+	case InternalKeyKindRangeDelete, InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
+		panic("unexpected range delete or range key kind in AddInternalKey")
+	case InternalKeyKindSingleDelete, InternalKeyKindDelete:
+		b.prepareDeferredKeyRecord(len(key.UserKey), key.Kind())
+	default:
+		b.prepareDeferredKeyValueRecord(keyLen, len(value), key.Kind())
+		hasValue = true
+	}
+	b.deferredOp.index = b.index
+	copy(b.deferredOp.Key, key.UserKey)
+	if hasValue {
+		copy(b.deferredOp.Value, value)
+	}
+	// TODO(peter): Manually inline DeferredBatchOp.Finish(). Mid-stack inlining
+	// in go1.13 will remove the need for this.
+	if b.index != nil {
+		if err := b.index.Add(b.deferredOp.offset); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Set adds an action to the batch that sets the key to map to the value.
 //
 // It is safe to modify the contents of the arguments after Set returns.
@@ -680,6 +783,72 @@ func (b *Batch) DeleteDeferred(keyLen int) *DeferredBatchOp {
 	return &b.deferredOp
 }
 
+// DeleteSized behaves identically to Delete, but takes an additional
+// argument indicating the size of the value being deleted. DeleteSized
+// should be preferred when the caller has the expectation that there exists
+// a single internal KV pair for the key (eg, the key has not been
+// overwritten recently), and the caller knows the size of its value.
+//
+// DeleteSized will record the value size within the tombstone and use it to
+// inform compaction-picking heuristics which strive to reduce space
+// amplification in the LSM. This "calling your shot" mechanic allows the
+// storage engine to more accurately estimate and reduce space amplification.
+//
+// It is safe to modify the contents of the arguments after DeleteSized
+// returns.
+func (b *Batch) DeleteSized(key []byte, deletedValueSize uint32, _ *WriteOptions) error {
+	deferredOp := b.DeleteSizedDeferred(len(key), deletedValueSize)
+	copy(b.deferredOp.Key, key)
+	// TODO(peter): Manually inline DeferredBatchOp.Finish(). Check if in a
+	// later Go release this is unnecessary.
+	if b.index != nil {
+		if err := b.index.Add(deferredOp.offset); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// DeleteSizedDeferred is similar to DeleteSized in that it adds a sized delete
+// operation to the batch, except it only takes in key length instead of a
+// complete key slice, letting the caller encode into the DeferredBatchOp.Key
+// slice and then call Finish() on the returned object.
+func (b *Batch) DeleteSizedDeferred(keyLen int, deletedValueSize uint32) *DeferredBatchOp {
+	if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete {
+		b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete
+	}
+
+	// Encode the sum of the key length and the value in the value.
+	v := uint64(deletedValueSize) + uint64(keyLen)
+
+	// Encode `v` as a varint.
+	var buf [binary.MaxVarintLen64]byte
+	n := 0
+	{
+		x := v
+		for x >= 0x80 {
+			buf[n] = byte(x) | 0x80
+			x >>= 7
+			n++
+		}
+		buf[n] = byte(x)
+		n++
+	}
+
+	// NB: In batch entries and sstable entries, values are stored as
+	// varstrings. Here, the value is itself a simple varint. This results in an
+	// unnecessary double layer of encoding:
+	//     varint(n) varint(deletedValueSize)
+	// The first varint will always be 1-byte, since a varint-encoded uint64
+	// will never exceed 128 bytes. This unnecessary extra byte and wrapping is
+	// preserved to avoid special casing across the database, and in particular
+	// in sstable block decoding which is performance sensitive.
+	b.prepareDeferredKeyValueRecord(keyLen, n, InternalKeyKindDeleteSized)
+	b.deferredOp.index = b.index
+	copy(b.deferredOp.Value, buf[:n])
+	return &b.deferredOp
+}
+
 // SingleDelete adds an action to the batch that single deletes the entry for key.
 // See Writer.SingleDelete for more details on the semantics of SingleDelete.
 //
@@ -782,6 +951,9 @@ func (b *Batch) rangeKeySetDeferred(startLen, internalValueLen int) *DeferredBat
 
 func (b *Batch) incrementRangeKeysCount() {
 	b.countRangeKeys++
+	if b.minimumFormatMajorVersion < FormatRangeKeys {
+		b.minimumFormatMajorVersion = FormatRangeKeys
+	}
 	if b.index != nil {
 		b.rangeKeys = nil
 		b.rangeKeysSeqNum = 0
@@ -895,6 +1067,7 @@ func (b *Batch) ingestSST(fileNum base.FileNum) {
 	// is not reset because for the InternalKeyKindIngestSST the count is the
 	// number of sstable paths which have been added to the batch.
 	b.memTableSize = origMemTableSize
+	b.minimumFormatMajorVersion = FormatFlushableIngest
 }
 
 // Empty returns true if the batch is empty, and false otherwise.
@@ -930,11 +1103,12 @@ func (b *Batch) SetRepr(data []byte) error {
 	}
 	b.data = data
 	b.count = uint64(binary.LittleEndian.Uint32(b.countData()))
+	var err error
 	if b.db != nil {
 		// Only track memTableSize for batches that will be committed to the DB.
-		b.refreshMemTableSize()
+		err = b.refreshMemTableSize()
 	}
-	return nil
+	return err
 }
 
 // NewIter returns an iterator that is unpositioned (Iterator.Valid() will
@@ -944,8 +1118,8 @@ func (b *Batch) SetRepr(data []byte) error {
 // The returned Iterator observes all of the Batch's existing mutations, but no
 // later mutations. Its view can be refreshed via RefreshBatchSnapshot or
 // SetOptions().
-func (b *Batch) NewIter(o *IterOptions) *Iterator {
+func (b *Batch) NewIter(o *IterOptions) (*Iterator, error) {
-	return b.NewIterWithContext(context.Background(), o)
+	return b.NewIterWithContext(context.Background(), o), nil
 }
 
 // NewIterWithContext is like NewIter, and additionally accepts a context for
@@ -954,7 +1128,7 @@ func (b *Batch) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterato
 	if b.index == nil {
 		return &Iterator{err: ErrNotIndexed}
 	}
-	return b.db.newIter(ctx, b, nil /* snapshot */, o)
+	return b.db.newIter(ctx, b, snapshotIterOpts{}, o)
 }
 
 // newInternalIter creates a new internalIterator that iterates over the
@@ -1167,38 +1341,40 @@ func (b *Batch) Indexed() bool {
 	return b.index != nil
 }
 
-func (b *Batch) init(cap int) {
+// init ensures that the batch data slice is initialized to meet the
+// minimum required size and allocates space for the batch header.
+func (b *Batch) init(size int) {
 	n := batchInitialSize
-	for n < cap {
+	for n < size {
 		n *= 2
 	}
-	b.data = rawalloc.New(batchHeaderLen, n)
+	if cap(b.data) < n {
-	b.setCount(0)
+		b.data = rawalloc.New(batchHeaderLen, n)
-	b.setSeqNum(0)
+	}
 	b.data = b.data[:batchHeaderLen]
+	// Zero the sequence number in the header.
+	for i := 0; i < len(b.data); i++ {
+		b.data[i] = 0
+	}
 }
 
 // Reset resets the batch for reuse. The underlying byte slice (that is
-// returned by Repr()) is not modified. It is only necessary to call this
+// returned by Repr()) may not be modified. It is only necessary to call this
 // method if a batch is explicitly being reused. Close automatically takes are
 // of releasing resources when appropriate for batches that are internally
 // being reused.
 func (b *Batch) Reset() {
-	b.count = 0
+	// Zero out the struct, retaining only the fields necessary for manual
-	b.countRangeDels = 0
+	// reuse.
-	b.countRangeKeys = 0
+	b.batchInternal = batchInternal{
-	b.memTableSize = 0
+		data:           b.data,
-	b.deferredOp = DeferredBatchOp{}
+		cmp:            b.cmp,
-	b.tombstones = nil
+		formatKey:      b.formatKey,
-	b.tombstonesSeqNum = 0
+		abbreviatedKey: b.abbreviatedKey,
-	b.rangeKeys = nil
+		index:          b.index,
-	b.rangeKeysSeqNum = 0
+		db:             b.db,
-	b.flushable = nil
+	}
-	b.commit = sync.WaitGroup{}
+	b.applied.Store(false)
-	b.fsyncWait = sync.WaitGroup{}
-	b.commitStats = BatchCommitStats{}
-	b.commitErr = nil
-	atomic.StoreUint32(&b.applied, 0)
 	if b.data != nil {
 		if cap(b.data) > batchMaxRetainedSize {
 			// If the capacity of the buffer is larger than our maximum
@@ -1209,13 +1385,14 @@ func (b *Batch) Reset() {
 		} else {
 			// Otherwise, reset the buffer for re-use.
 			b.data = b.data[:batchHeaderLen]
-			b.setSeqNum(0)
+			// Zero the sequence number in the header.
+			for i := 0; i < len(b.data); i++ {
+				b.data[i] = 0
+			}
 		}
 	}
 	if b.index != nil {
 		b.index.Init(&b.data, b.cmp, b.abbreviatedKey)
-		b.rangeDelIndex = nil
-		b.rangeKeyIndex = nil
 	}
 }
 
@@ -1287,6 +1464,11 @@ func (b *Batch) Reader() BatchReader {
 }
 
 func batchDecodeStr(data []byte) (odata []byte, s []byte, ok bool) {
+	// TODO(jackson): This will index out of bounds if there's no varint or an
+	// invalid varint (eg, a single 0xff byte). Correcting will add a bit of
+	// overhead. We could avoid that overhead whenever len(data) >=
+	// binary.MaxVarint32?
+
 	var v uint32
 	var n int
 	ptr := unsafe.Pointer(&data[0])
@@ -1349,29 +1531,32 @@ func ReadBatch(repr []byte) (r BatchReader, count uint32) {
 	return repr[batchHeaderLen:], count
 }
 
-// Next returns the next entry in this batch. The final return value is false
+// Next returns the next entry in this batch, if there is one. If the reader has
-// if the batch is corrupt. The end of batch is reached when len(r)==0.
+// reached the end of the batch, Next returns ok=false and a nil error. If the
-func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, ok bool) {
+// batch is corrupt and the next entry is illegible, Next returns ok=false and a
+// non-nil error.
+func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, ok bool, err error) {
 	if len(*r) == 0 {
-		return 0, nil, nil, false
+		return 0, nil, nil, false, nil
 	}
 	kind = InternalKeyKind((*r)[0])
 	if kind > InternalKeyKindMax {
-		return 0, nil, nil, false
+		return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "invalid key kind 0x%x", (*r)[0])
 	}
 	*r, ukey, ok = batchDecodeStr((*r)[1:])
 	if !ok {
-		return 0, nil, nil, false
+		return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "decoding user key")
 	}
 	switch kind {
 	case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete,
-		InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
+		InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete,
+		InternalKeyKindDeleteSized:
 		*r, value, ok = batchDecodeStr(*r)
 		if !ok {
-			return 0, nil, nil, false
+			return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "decoding %s value", kind)
 		}
 	}
-	return kind, ukey, value, true
+	return kind, ukey, value, true, nil
 }
 
 // Note: batchIter mirrors the implementation of flushableBatchIter. Keep the
@@ -1500,7 +1685,8 @@ func (i *batchIter) value() []byte {
 
 	switch InternalKeyKind(data[offset]) {
 	case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete,
-		InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
+		InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete,
+		InternalKeyKindDeleteSized:
 		_, value, ok := batchDecodeStr(data[keyEnd:])
 		if !ok {
 			return nil
@@ -1572,7 +1758,7 @@ var _ flushable = (*flushableBatch)(nil)
 // interface. This allows the batch to act like a memtable and be placed in the
 // queue of flushable memtables. Note that the flushable batch takes ownership
 // of the batch data.
-func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
+func newFlushableBatch(batch *Batch, comparer *Comparer) (*flushableBatch, error) {
 	b := &flushableBatch{
 		data:      batch.data,
 		cmp:       comparer.Compare,
@@ -1593,8 +1779,11 @@ func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
 		var index uint32
 		for iter := BatchReader(b.data[batchHeaderLen:]); len(iter) > 0; index++ {
 			offset := uintptr(unsafe.Pointer(&iter[0])) - uintptr(unsafe.Pointer(&b.data[0]))
-			kind, key, _, ok := iter.Next()
+			kind, key, _, ok, err := iter.Next()
 			if !ok {
+				if err != nil {
+					return nil, err
+				}
 				break
 			}
 			entry := flushableBatchEntry{
@@ -1666,7 +1855,7 @@ func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
 		}
 		fragmentRangeKeys(frag, it, len(rangeKeyOffsets))
 	}
-	return b
+	return b, nil
 }
 
 func (b *flushableBatch) setSeqNum(seqNum uint64) {
@@ -1715,6 +1904,7 @@ func (b *flushableBatch) Swap(i, j int) {
 	b.offsets[i], b.offsets[j] = b.offsets[j], b.offsets[i]
 }
 
+// newIter is part of the flushable interface.
 func (b *flushableBatch) newIter(o *IterOptions) internalIterator {
 	return &flushableBatchIter{
 		batch:   b,
@@ -1727,6 +1917,7 @@ func (b *flushableBatch) newIter(o *IterOptions) internalIterator {
 	}
 }
 
+// newFlushIter is part of the flushable interface.
 func (b *flushableBatch) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
 	return &flushFlushableBatchIter{
 		flushableBatchIter: flushableBatchIter{
@@ -1740,6 +1931,7 @@ func (b *flushableBatch) newFlushIter(o *IterOptions, bytesFlushed *uint64) inte
 	}
 }
 
+// newRangeDelIter is part of the flushable interface.
 func (b *flushableBatch) newRangeDelIter(o *IterOptions) keyspan.FragmentIterator {
 	if len(b.tombstones) == 0 {
 		return nil
@@ -1747,6 +1939,7 @@ func (b *flushableBatch) newRangeDelIter(o *IterOptions) keyspan.FragmentIterato
 	return keyspan.NewIter(b.cmp, b.tombstones)
 }
 
+// newRangeKeyIter is part of the flushable interface.
 func (b *flushableBatch) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator {
 	if len(b.rangeKeys) == 0 {
 		return nil
@@ -1754,17 +1947,23 @@ func (b *flushableBatch) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterato
 	return keyspan.NewIter(b.cmp, b.rangeKeys)
 }
 
+// containsRangeKeys is part of the flushable interface.
 func (b *flushableBatch) containsRangeKeys() bool { return len(b.rangeKeys) > 0 }
 
+// inuseBytes is part of the flushable interface.
 func (b *flushableBatch) inuseBytes() uint64 {
 	return uint64(len(b.data) - batchHeaderLen)
 }
 
+// totalBytes is part of the flushable interface.
 func (b *flushableBatch) totalBytes() uint64 {
 	return uint64(cap(b.data))
 }
 
+// readyForFlush is part of the flushable interface.
 func (b *flushableBatch) readyForFlush() bool {
+	// A flushable batch is always ready for flush; it must be flushed together
+	// with the previous memtable.
 	return true
 }
 
@@ -1944,7 +2143,8 @@ func (i *flushableBatchIter) value() base.LazyValue {
 	var ok bool
 	switch kind {
 	case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete,
-		InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
+		InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete,
+		InternalKeyKindDeleteSized:
 		keyEnd := i.offsets[i.index].keyEnd
 		_, value, ok = batchDecodeStr(i.data[keyEnd:])
 		if !ok {
@@ -2081,7 +2281,10 @@ func batchSort(
 		rangeKeyIter := b.newRangeKeyIter(nil, math.MaxUint64)
 		return pointIter, rangeDelIter, rangeKeyIter
 	}
-	f := newFlushableBatch(b, b.db.opts.Comparer)
+	f, err := newFlushableBatch(b, b.db.opts.Comparer)
+	if err != nil {
+		panic(err)
+	}
 	return f.newIter(nil), f.newRangeDelIter(nil), f.newRangeKeyIter(nil)
 }
 

vendor/github.com/cockroachdb/pebble/checkpoint.go

@@ -188,7 +188,10 @@ func (d *DB) Checkpoint(
 	manifestFileNum := d.mu.versions.manifestFileNum
 	manifestSize := d.mu.versions.manifest.Size()
 	optionsFileNum := d.optionsFileNum
-
+	virtualBackingFiles := make(map[base.DiskFileNum]struct{})
+	for diskFileNum := range d.mu.versions.backingState.fileBackingMap {
+		virtualBackingFiles[diskFileNum] = struct{}{}
+	}
 	// Release the manifest and DB.mu so we don't block other operations on
 	// the database.
 	d.mu.versions.logUnlock()
@@ -209,11 +212,7 @@ func (d *DB) Checkpoint(
 		}
 		if ckErr != nil {
 			// Attempt to cleanup on error.
-			paths, _ := fs.List(destDir)
+			_ = fs.RemoveAll(destDir)
-			for _, path := range paths {
-				_ = fs.Remove(path)
-			}
-			_ = fs.Remove(destDir)
 		}
 	}()
 	dir, ckErr = mkdirAllAndSyncParents(fs, destDir)
@@ -254,7 +253,9 @@ func (d *DB) Checkpoint(
 	}
 
 	var excludedFiles map[deletedFileEntry]*fileMetadata
-
+	// Set of FileBacking.DiskFileNum which will be required by virtual sstables
+	// in the checkpoint.
+	requiredVirtualBackingFiles := make(map[base.DiskFileNum]struct{})
 	// Link or copy the sstables.
 	for l := range current.Levels {
 		iter := current.Levels[l].Iter()
@@ -270,7 +271,15 @@ func (d *DB) Checkpoint(
 				continue
 			}
 
-			srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, f.FileNum)
+			fileBacking := f.FileBacking
+			if f.Virtual {
+				if _, ok := requiredVirtualBackingFiles[fileBacking.DiskFileNum]; ok {
+					continue
+				}
+				requiredVirtualBackingFiles[fileBacking.DiskFileNum] = struct{}{}
+			}
+
+			srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, fileBacking.DiskFileNum)
 			destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
 			ckErr = vfs.LinkOrCopy(fs, srcPath, destPath)
 			if ckErr != nil {
@@ -279,7 +288,19 @@ func (d *DB) Checkpoint(
 		}
 	}
 
-	ckErr = d.writeCheckpointManifest(fs, formatVers, destDir, dir, manifestFileNum, manifestSize, excludedFiles)
+	var removeBackingTables []base.DiskFileNum
+	for diskFileNum := range virtualBackingFiles {
+		if _, ok := requiredVirtualBackingFiles[diskFileNum]; !ok {
+			// The backing sstable associated with fileNum is no longer
+			// required.
+			removeBackingTables = append(removeBackingTables, diskFileNum)
+		}
+	}
+
+	ckErr = d.writeCheckpointManifest(
+		fs, formatVers, destDir, dir, manifestFileNum.DiskFileNum(), manifestSize,
+		excludedFiles, removeBackingTables,
+	)
 	if ckErr != nil {
 		return ckErr
 	}
@@ -292,7 +313,7 @@ func (d *DB) Checkpoint(
 		if logNum == 0 {
 			continue
 		}
-		srcPath := base.MakeFilepath(fs, d.walDirname, fileTypeLog, logNum)
+		srcPath := base.MakeFilepath(fs, d.walDirname, fileTypeLog, logNum.DiskFileNum())
 		destPath := fs.PathJoin(destDir, fs.PathBase(srcPath))
 		ckErr = vfs.Copy(fs, srcPath, destPath)
 		if ckErr != nil {
@@ -315,9 +336,10 @@ func (d *DB) writeCheckpointManifest(
 	formatVers FormatMajorVersion,
 	destDirPath string,
 	destDir vfs.File,
-	manifestFileNum FileNum,
+	manifestFileNum base.DiskFileNum,
 	manifestSize int64,
 	excludedFiles map[deletedFileEntry]*fileMetadata,
+	removeBackingTables []base.DiskFileNum,
 ) error {
 	// Copy the MANIFEST, and create a pointer to it. We copy rather
 	// than link because additional version edits added to the
@@ -346,7 +368,7 @@ func (d *DB) writeCheckpointManifest(
 		// need to append another record with the excluded files (we cannot simply
 		// append a record after a raw data copy; see
 		// https://github.com/cockroachdb/cockroach/issues/100935).
-		r := record.NewReader(&io.LimitedReader{R: src, N: manifestSize}, manifestFileNum)
+		r := record.NewReader(&io.LimitedReader{R: src, N: manifestSize}, manifestFileNum.FileNum())
 		w := record.NewWriter(dst)
 		for {
 			rr, err := r.Next()
@@ -369,8 +391,10 @@ func (d *DB) writeCheckpointManifest(
 		if len(excludedFiles) > 0 {
 			// Write out an additional VersionEdit that deletes the excluded SST files.
 			ve := versionEdit{
-				DeletedFiles: excludedFiles,
+				DeletedFiles:         excludedFiles,
+				RemovedBackingTables: removeBackingTables,
 			}
+
 			rw, err := w.Next()
 			if err != nil {
 				return err
@@ -397,7 +421,7 @@ func (d *DB) writeCheckpointManifest(
 	if err != nil {
 		return err
 	}
-	if err := setCurrentFunc(formatVers, manifestMarker, fs, destDirPath, destDir)(manifestFileNum); err != nil {
+	if err := setCurrentFunc(formatVers, manifestMarker, fs, destDirPath, destDir)(manifestFileNum.FileNum()); err != nil {
 		return err
 	}
 	return manifestMarker.Close()

vendor/github.com/cockroachdb/pebble/cleaner.go

@@ -4,7 +4,18 @@
 
 package pebble
 
-import "github.com/cockroachdb/pebble/internal/base"
+import (
+	"context"
+	"runtime/pprof"
+	"sync"
+	"time"
+
+	"github.com/cockroachdb/errors/oserror"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/tokenbucket"
+)
 
 // Cleaner exports the base.Cleaner type.
 type Cleaner = base.Cleaner
@@ -14,3 +25,271 @@ type DeleteCleaner = base.DeleteCleaner
 
 // ArchiveCleaner exports the base.ArchiveCleaner type.
 type ArchiveCleaner = base.ArchiveCleaner
+
+type cleanupManager struct {
+	opts            *Options
+	objProvider     objstorage.Provider
+	onTableDeleteFn func(fileSize uint64)
+	deletePacer     *deletionPacer
+
+	// jobsCh is used as the cleanup job queue.
+	jobsCh chan *cleanupJob
+	// waitGroup is used to wait for the background goroutine to exit.
+	waitGroup sync.WaitGroup
+
+	mu struct {
+		sync.Mutex
+		// totalJobs is the total number of enqueued jobs (completed or in progress).
+		totalJobs              int
+		completedJobs          int
+		completedJobsCond      sync.Cond
+		jobsQueueWarningIssued bool
+	}
+}
+
+// We can queue this many jobs before we have to block EnqueueJob.
+const jobsQueueDepth = 1000
+
+// obsoleteFile holds information about a file that needs to be deleted soon.
+type obsoleteFile struct {
+	dir      string
+	fileNum  base.DiskFileNum
+	fileType fileType
+	fileSize uint64
+}
+
+type cleanupJob struct {
+	jobID         int
+	obsoleteFiles []obsoleteFile
+}
+
+// openCleanupManager creates a cleanupManager and starts its background goroutine.
+// The cleanupManager must be Close()d.
+func openCleanupManager(
+	opts *Options,
+	objProvider objstorage.Provider,
+	onTableDeleteFn func(fileSize uint64),
+	getDeletePacerInfo func() deletionPacerInfo,
+) *cleanupManager {
+	cm := &cleanupManager{
+		opts:            opts,
+		objProvider:     objProvider,
+		onTableDeleteFn: onTableDeleteFn,
+		deletePacer:     newDeletionPacer(time.Now(), int64(opts.TargetByteDeletionRate), getDeletePacerInfo),
+		jobsCh:          make(chan *cleanupJob, jobsQueueDepth),
+	}
+	cm.mu.completedJobsCond.L = &cm.mu.Mutex
+	cm.waitGroup.Add(1)
+
+	go func() {
+		pprof.Do(context.Background(), gcLabels, func(context.Context) {
+			cm.mainLoop()
+		})
+	}()
+
+	return cm
+}
+
+// Close stops the background goroutine, waiting until all queued jobs are completed.
+// Delete pacing is disabled for the remaining jobs.
+func (cm *cleanupManager) Close() {
+	close(cm.jobsCh)
+	cm.waitGroup.Wait()
+}
+
+// EnqueueJob adds a cleanup job to the manager's queue.
+func (cm *cleanupManager) EnqueueJob(jobID int, obsoleteFiles []obsoleteFile) {
+	job := &cleanupJob{
+		jobID:         jobID,
+		obsoleteFiles: obsoleteFiles,
+	}
+
+	// Report deleted bytes to the pacer, which can use this data to potentially
+	// increase the deletion rate to keep up. We want to do this at enqueue time
+	// rather than when we get to the job, otherwise the reported bytes will be
+	// subject to the throttling rate which defeats the purpose.
+	var pacingBytes uint64
+	for _, of := range obsoleteFiles {
+		if cm.needsPacing(of.fileType, of.fileNum) {
+			pacingBytes += of.fileSize
+		}
+	}
+	if pacingBytes > 0 {
+		cm.deletePacer.ReportDeletion(time.Now(), pacingBytes)
+	}
+
+	cm.mu.Lock()
+	cm.mu.totalJobs++
+	cm.maybeLogLocked()
+	cm.mu.Unlock()
+
+	if invariants.Enabled && len(cm.jobsCh) >= cap(cm.jobsCh)-2 {
+		panic("cleanup jobs queue full")
+	}
+
+	cm.jobsCh <- job
+}
+
+// Wait until the completion of all jobs that were already queued.
+//
+// Does not wait for jobs that are enqueued during the call.
+//
+// Note that DB.mu should not be held while calling this method; the background
+// goroutine needs to acquire DB.mu to update deleted table metrics.
+func (cm *cleanupManager) Wait() {
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+	n := cm.mu.totalJobs
+	for cm.mu.completedJobs < n {
+		cm.mu.completedJobsCond.Wait()
+	}
+}
+
+// mainLoop runs the manager's background goroutine.
+func (cm *cleanupManager) mainLoop() {
+	defer cm.waitGroup.Done()
+
+	var tb tokenbucket.TokenBucket
+	// Use a token bucket with 1 token / second refill rate and 1 token burst.
+	tb.Init(1.0, 1.0)
+	for job := range cm.jobsCh {
+		for _, of := range job.obsoleteFiles {
+			if of.fileType != fileTypeTable {
+				path := base.MakeFilepath(cm.opts.FS, of.dir, of.fileType, of.fileNum)
+				cm.deleteObsoleteFile(of.fileType, job.jobID, path, of.fileNum, of.fileSize)
+			} else {
+				cm.maybePace(&tb, of.fileType, of.fileNum, of.fileSize)
+				cm.onTableDeleteFn(of.fileSize)
+				cm.deleteObsoleteObject(fileTypeTable, job.jobID, of.fileNum)
+			}
+		}
+		cm.mu.Lock()
+		cm.mu.completedJobs++
+		cm.mu.completedJobsCond.Broadcast()
+		cm.maybeLogLocked()
+		cm.mu.Unlock()
+	}
+}
+
+func (cm *cleanupManager) needsPacing(fileType base.FileType, fileNum base.DiskFileNum) bool {
+	if fileType != fileTypeTable {
+		return false
+	}
+	meta, err := cm.objProvider.Lookup(fileType, fileNum)
+	if err != nil {
+		// The object was already removed from the provider; we won't actually
+		// delete anything, so we don't need to pace.
+		return false
+	}
+	// Don't throttle deletion of remote objects.
+	return !meta.IsRemote()
+}
+
+// maybePace sleeps before deleting an object if appropriate. It is always
+// called from the background goroutine.
+func (cm *cleanupManager) maybePace(
+	tb *tokenbucket.TokenBucket, fileType base.FileType, fileNum base.DiskFileNum, fileSize uint64,
+) {
+	if !cm.needsPacing(fileType, fileNum) {
+		return
+	}
+
+	tokens := cm.deletePacer.PacingDelay(time.Now(), fileSize)
+	if tokens == 0.0 {
+		// The token bucket might be in debt; it could make us wait even for 0
+		// tokens. We don't want that if the pacer decided throttling should be
+		// disabled.
+		return
+	}
+	// Wait for tokens. We use a token bucket instead of sleeping outright because
+	// the token bucket accumulates up to one second of unused tokens.
+	for {
+		ok, d := tb.TryToFulfill(tokenbucket.Tokens(tokens))
+		if ok {
+			break
+		}
+		time.Sleep(d)
+	}
+}
+
+// deleteObsoleteFile deletes a (non-object) file that is no longer needed.
+func (cm *cleanupManager) deleteObsoleteFile(
+	fileType fileType, jobID int, path string, fileNum base.DiskFileNum, fileSize uint64,
+) {
+	// TODO(peter): need to handle this error, probably by re-adding the
+	// file that couldn't be deleted to one of the obsolete slices map.
+	err := cm.opts.Cleaner.Clean(cm.opts.FS, fileType, path)
+	if oserror.IsNotExist(err) {
+		return
+	}
+
+	switch fileType {
+	case fileTypeLog:
+		cm.opts.EventListener.WALDeleted(WALDeleteInfo{
+			JobID:   jobID,
+			Path:    path,
+			FileNum: fileNum.FileNum(),
+			Err:     err,
+		})
+	case fileTypeManifest:
+		cm.opts.EventListener.ManifestDeleted(ManifestDeleteInfo{
+			JobID:   jobID,
+			Path:    path,
+			FileNum: fileNum.FileNum(),
+			Err:     err,
+		})
+	case fileTypeTable:
+		panic("invalid deletion of object file")
+	}
+}
+
+func (cm *cleanupManager) deleteObsoleteObject(
+	fileType fileType, jobID int, fileNum base.DiskFileNum,
+) {
+	if fileType != fileTypeTable {
+		panic("not an object")
+	}
+
+	var path string
+	meta, err := cm.objProvider.Lookup(fileType, fileNum)
+	if err != nil {
+		path = "<nil>"
+	} else {
+		path = cm.objProvider.Path(meta)
+		err = cm.objProvider.Remove(fileType, fileNum)
+	}
+	if cm.objProvider.IsNotExistError(err) {
+		return
+	}
+
+	switch fileType {
+	case fileTypeTable:
+		cm.opts.EventListener.TableDeleted(TableDeleteInfo{
+			JobID:   jobID,
+			Path:    path,
+			FileNum: fileNum.FileNum(),
+			Err:     err,
+		})
+	}
+}
+
+// maybeLogLocked issues a log if the job queue gets 75% full and issues a log
+// when the job queue gets back to less than 10% full.
+//
+// Must be called with cm.mu locked.
+func (cm *cleanupManager) maybeLogLocked() {
+	const highThreshold = jobsQueueDepth * 3 / 4
+	const lowThreshold = jobsQueueDepth / 10
+
+	jobsInQueue := cm.mu.totalJobs - cm.mu.completedJobs
+
+	if !cm.mu.jobsQueueWarningIssued && jobsInQueue > highThreshold {
+		cm.mu.jobsQueueWarningIssued = true
+		cm.opts.Logger.Infof("cleanup falling behind; job queue has over %d jobs", highThreshold)
+	}
+
+	if cm.mu.jobsQueueWarningIssued && jobsInQueue < lowThreshold {
+		cm.mu.jobsQueueWarningIssued = false
+		cm.opts.Logger.Infof("cleanup back to normal; job queue has under %d jobs", lowThreshold)
+	}
+}

vendor/github.com/cockroachdb/pebble/commit.go

@@ -9,7 +9,6 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
-	"unsafe"
 
 	"github.com/cockroachdb/pebble/record"
 )
@@ -33,13 +32,13 @@ type commitQueue struct {
 	//
 	// The head index is stored in the most-significant bits so that we can
 	// atomically add to it and the overflow is harmless.
-	headTail uint64
+	headTail atomic.Uint64
 
 	// slots is a ring buffer of values stored in this queue. The size must be a
 	// power of 2. A slot is in use until *both* the tail index has moved beyond
 	// it and the slot value has been set to nil. The slot value is set to nil
 	// atomically by the consumer and read atomically by the producer.
-	slots [record.SyncConcurrency]unsafe.Pointer
+	slots [record.SyncConcurrency]atomic.Pointer[Batch]
 }
 
 const dequeueBits = 32
@@ -58,7 +57,7 @@ func (q *commitQueue) pack(head, tail uint32) uint64 {
 }
 
 func (q *commitQueue) enqueue(b *Batch) {
-	ptrs := atomic.LoadUint64(&q.headTail)
+	ptrs := q.headTail.Load()
 	head, tail := q.unpack(ptrs)
 	if (tail+uint32(len(q.slots)))&(1<<dequeueBits-1) == head {
 		// Queue is full. This should never be reached because commitPipeline.commitQueueSem
@@ -67,8 +66,8 @@ func (q *commitQueue) enqueue(b *Batch) {
 	}
 	slot := &q.slots[head&uint32(len(q.slots)-1)]
 
-	// Check if the head slot has been released by dequeue.
+	// Check if the head slot has been released by dequeueApplied.
-	for atomic.LoadPointer(slot) != nil {
+	for slot.Load() != nil {
 		// Another goroutine is still cleaning up the tail, so the queue is
 		// actually still full. We spin because this should resolve itself
 		// momentarily.
@@ -76,16 +75,20 @@ func (q *commitQueue) enqueue(b *Batch) {
 	}
 
 	// The head slot is free, so we own it.
-	atomic.StorePointer(slot, unsafe.Pointer(b))
+	slot.Store(b)
 
-	// Increment head. This passes ownership of slot to dequeue and acts as a
+	// Increment head. This passes ownership of slot to dequeueApplied and acts as a
 	// store barrier for writing the slot.
-	atomic.AddUint64(&q.headTail, 1<<dequeueBits)
+	q.headTail.Add(1 << dequeueBits)
 }
 
-func (q *commitQueue) dequeue() *Batch {
+// dequeueApplied removes the earliest enqueued Batch, if it is applied.
+//
+// Returns nil if the commit queue is empty or the earliest Batch is not yet
+// applied.
+func (q *commitQueue) dequeueApplied() *Batch {
 	for {
-		ptrs := atomic.LoadUint64(&q.headTail)
+		ptrs := q.headTail.Load()
 		head, tail := q.unpack(ptrs)
 		if tail == head {
 			// Queue is empty.
@@ -93,8 +96,8 @@ func (q *commitQueue) dequeue() *Batch {
 		}
 
 		slot := &q.slots[tail&uint32(len(q.slots)-1)]
-		b := (*Batch)(atomic.LoadPointer(slot))
+		b := slot.Load()
-		if b == nil || atomic.LoadUint32(&b.applied) == 0 {
+		if b == nil || !b.applied.Load() {
 			// The batch is not ready to be dequeued, or another goroutine has
 			// already dequeued it.
 			return nil
@@ -103,13 +106,13 @@ func (q *commitQueue) dequeue() *Batch {
 		// Confirm head and tail (for our speculative check above) and increment
 		// tail. If this succeeds, then we own the slot at tail.
 		ptrs2 := q.pack(head, tail+1)
-		if atomic.CompareAndSwapUint64(&q.headTail, ptrs, ptrs2) {
+		if q.headTail.CompareAndSwap(ptrs, ptrs2) {
 			// We now own slot.
 			//
 			// Tell enqueue that we're done with this slot. Zeroing the slot is also
 			// important so we don't leave behind references that could keep this object
 			// live longer than necessary.
-			atomic.StorePointer(slot, nil)
+			slot.Store(nil)
 			// At this point enqueue owns the slot.
 			return b
 		}
@@ -122,10 +125,10 @@ func (q *commitQueue) dequeue() *Batch {
 type commitEnv struct {
 	// The next sequence number to give to a batch. Protected by
 	// commitPipeline.mu.
-	logSeqNum *uint64
+	logSeqNum *atomic.Uint64
 	// The visible sequence number at which reads should be performed. Ratcheted
 	// upwards atomically as batches are applied to the memtable.
-	visibleSeqNum *uint64
+	visibleSeqNum *atomic.Uint64
 
 	// Apply the batch to the specified memtable. Called concurrently.
 	apply func(b *Batch, mem *memTable) error
@@ -239,6 +242,7 @@ type commitPipeline struct {
 	// logSyncQSem are used for this reservation.
 	commitQueueSem chan struct{}
 	logSyncQSem    chan struct{}
+	ingestSem      chan struct{}
 	// The mutex to use for synchronizing access to logSeqNum and serializing
 	// calls to commitEnv.write().
 	mu sync.Mutex
@@ -263,6 +267,7 @@ func newCommitPipeline(env commitEnv) *commitPipeline {
 		// and sync the WAL.
 		commitQueueSem: make(chan struct{}, record.SyncConcurrency-1),
 		logSyncQSem:    make(chan struct{}, record.SyncConcurrency-1),
+		ingestSem:      make(chan struct{}, 1),
 	}
 	return p
 }
@@ -381,12 +386,12 @@ func (p *commitPipeline) AllocateSeqNum(
 	// Assign the batch a sequence number. Note that we use atomic operations
 	// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
 	// mutual exclusion for other goroutines writing to logSeqNum.
-	logSeqNum := atomic.AddUint64(p.env.logSeqNum, uint64(count)) - uint64(count)
+	logSeqNum := p.env.logSeqNum.Add(uint64(count)) - uint64(count)
 	seqNum := logSeqNum
 	if seqNum == 0 {
 		// We can't use the value 0 for the global seqnum during ingestion, because
 		// 0 indicates no global seqnum. So allocate one more seqnum.
-		atomic.AddUint64(p.env.logSeqNum, 1)
+		p.env.logSeqNum.Add(1)
 		seqNum++
 	}
 	b.setSeqNum(seqNum)
@@ -396,7 +401,7 @@ func (p *commitPipeline) AllocateSeqNum(
 	// writes that were sequenced before the ingestion. The spin loop is
 	// unfortunate, but obviates the need for additional synchronization.
 	for {
-		visibleSeqNum := atomic.LoadUint64(p.env.visibleSeqNum)
+		visibleSeqNum := p.env.visibleSeqNum.Load()
 		if visibleSeqNum == logSeqNum {
 			break
 		}
@@ -455,7 +460,7 @@ func (p *commitPipeline) prepare(b *Batch, syncWAL bool, noSyncWait bool) (*memT
 	// Assign the batch a sequence number. Note that we use atomic operations
 	// here to handle concurrent reads of logSeqNum. commitPipeline.mu provides
 	// mutual exclusion for other goroutines writing to logSeqNum.
-	b.setSeqNum(atomic.AddUint64(p.env.logSeqNum, n) - n)
+	b.setSeqNum(p.env.logSeqNum.Add(n) - n)
 
 	// Write the data to the WAL.
 	mem, err := p.env.write(b, syncWG, syncErr)
@@ -467,17 +472,17 @@ func (p *commitPipeline) prepare(b *Batch, syncWAL bool, noSyncWait bool) (*memT
 
 func (p *commitPipeline) publish(b *Batch) {
 	// Mark the batch as applied.
-	atomic.StoreUint32(&b.applied, 1)
+	b.applied.Store(true)
 
 	// Loop dequeuing applied batches from the pending queue. If our batch was
 	// the head of the pending queue we are guaranteed that either we'll publish
-	// it or someone else will dequeue and publish it. If our batch is not the
+	// it or someone else will dequeueApplied and publish it. If our batch is not the
-	// head of the queue then either we'll dequeue applied batches and reach our
+	// head of the queue then either we'll dequeueApplied applied batches and reach our
 	// batch or there is an unapplied batch blocking us. When that unapplied
 	// batch applies it will go through the same process and publish our batch
 	// for us.
 	for {
-		t := p.pending.dequeue()
+		t := p.pending.dequeueApplied()
 		if t == nil {
 			// Wait for another goroutine to publish us. We might also be waiting for
 			// the WAL sync to finish.
@@ -486,7 +491,7 @@ func (p *commitPipeline) publish(b *Batch) {
 			b.commitStats.CommitWaitDuration += time.Since(now)
 			break
 		}
-		if atomic.LoadUint32(&t.applied) != 1 {
+		if !t.applied.Load() {
 			panic("not reached")
 		}
 
@@ -495,13 +500,13 @@ func (p *commitPipeline) publish(b *Batch) {
 		// number for a subsequent batch. That's ok as all we're guaranteeing is
 		// that the sequence number ratchets up.
 		for {
-			curSeqNum := atomic.LoadUint64(p.env.visibleSeqNum)
+			curSeqNum := p.env.visibleSeqNum.Load()
 			newSeqNum := t.SeqNum() + uint64(t.Count())
 			if newSeqNum <= curSeqNum {
 				// t's sequence number has already been published.
 				break
 			}
-			if atomic.CompareAndSwapUint64(p.env.visibleSeqNum, curSeqNum, newSeqNum) {
+			if p.env.visibleSeqNum.CompareAndSwap(curSeqNum, newSeqNum) {
 				// We successfully published t's sequence number.
 				break
 			}

vendor/github.com/cockroachdb/pebble/compaction_iter.go

@@ -6,6 +6,7 @@ package pebble
 
 import (
 	"bytes"
+	"encoding/binary"
 	"fmt"
 	"io"
 	"sort"
@@ -14,9 +15,9 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/bytealloc"
-	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/rangekey"
+	"github.com/cockroachdb/redact"
 )
 
 // compactionIter provides a forward-only iterator that encapsulates the logic
@@ -205,7 +206,29 @@ type compactionIter struct {
 	// compaction iterator was only returned because an open snapshot prevents
 	// its elision. This field only applies to point keys, and not to range
 	// deletions or range keys.
+	//
+	// For MERGE, it is possible that doing the merge is interrupted even when
+	// the next point key is in the same stripe. This can happen if the loop in
+	// mergeNext gets interrupted by sameStripeNonSkippable.
+	// sameStripeNonSkippable occurs due to RANGEDELs that sort before
+	// SET/MERGE/DEL with the same seqnum, so the RANGEDEL does not necessarily
+	// delete the subsequent SET/MERGE/DEL keys.
 	snapshotPinned bool
+	// forceObsoleteDueToRangeDel is set to true in a subset of the cases that
+	// snapshotPinned is true. This value is true when the point is obsolete due
+	// to a RANGEDEL but could not be deleted due to a snapshot.
+	//
+	// NB: it may seem that the additional cases that snapshotPinned captures
+	// are harmless in that they can also be used to mark a point as obsolete
+	// (it is merely a duplication of some logic that happens in
+	// Writer.AddWithForceObsolete), but that is not quite accurate as of this
+	// writing -- snapshotPinned originated in stats collection and for a
+	// sequence MERGE, SET, where the MERGE cannot merge with the (older) SET
+	// due to a snapshot, the snapshotPinned value for the SET is true.
+	//
+	// TODO(sumeer,jackson): improve the logic of snapshotPinned and reconsider
+	// whether we need forceObsoleteDueToRangeDel.
+	forceObsoleteDueToRangeDel bool
 	// The index of the snapshot for the current key within the snapshots slice.
 	curSnapshotIdx    int
 	curSnapshotSeqNum uint64
@@ -233,13 +256,19 @@ type compactionIter struct {
 	// The fragmented range keys.
 	rangeKeys []keyspan.Span
 	// Byte allocator for the tombstone keys.
-	alloc               bytealloc.A
+	alloc                                  bytealloc.A
-	allowZeroSeqNum     bool
+	allowZeroSeqNum                        bool
-	elideTombstone      func(key []byte) bool
+	elideTombstone                         func(key []byte) bool
-	elideRangeTombstone func(start, end []byte) bool
+	elideRangeTombstone                    func(start, end []byte) bool
+	ineffectualSingleDeleteCallback        func(userKey []byte)
+	singleDeleteInvariantViolationCallback func(userKey []byte)
 	// The on-disk format major version. This informs the types of keys that
 	// may be written to disk during a compaction.
 	formatVersion FormatMajorVersion
+	stats         struct {
+		// count of DELSIZED keys that were missized.
+		countMissizedDels uint64
+	}
 }
 
 func newCompactionIter(
@@ -254,20 +283,24 @@ func newCompactionIter(
 	allowZeroSeqNum bool,
 	elideTombstone func(key []byte) bool,
 	elideRangeTombstone func(start, end []byte) bool,
+	ineffectualSingleDeleteCallback func(userKey []byte),
+	singleDeleteInvariantViolationCallback func(userKey []byte),
 	formatVersion FormatMajorVersion,
 ) *compactionIter {
 	i := &compactionIter{
-		equal:               equal,
+		equal:                                  equal,
-		merge:               merge,
+		merge:                                  merge,
-		iter:                iter,
+		iter:                                   iter,
-		snapshots:           snapshots,
+		snapshots:                              snapshots,
-		frontiers:           frontiers{cmp: cmp},
+		frontiers:                              frontiers{cmp: cmp},
-		rangeDelFrag:        rangeDelFrag,
+		rangeDelFrag:                           rangeDelFrag,
-		rangeKeyFrag:        rangeKeyFrag,
+		rangeKeyFrag:                           rangeKeyFrag,
-		allowZeroSeqNum:     allowZeroSeqNum,
+		allowZeroSeqNum:                        allowZeroSeqNum,
-		elideTombstone:      elideTombstone,
+		elideTombstone:                         elideTombstone,
-		elideRangeTombstone: elideRangeTombstone,
+		elideRangeTombstone:                    elideRangeTombstone,
-		formatVersion:       formatVersion,
+		ineffectualSingleDeleteCallback:        ineffectualSingleDeleteCallback,
+		singleDeleteInvariantViolationCallback: singleDeleteInvariantViolationCallback,
+		formatVersion:                          formatVersion,
 	}
 	i.rangeDelFrag.Cmp = cmp
 	i.rangeDelFrag.Format = formatKey
@@ -306,21 +339,43 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 		return nil, nil
 	}
 
-	// Prior to this call to `Next()` we are in one of three situations with
+	// Prior to this call to `Next()` we are in one of four situations with
 	// respect to `iterKey` and related state:
 	//
 	// - `!skip && pos == iterPosNext`: `iterKey` is already at the next key.
-	// - `!skip && pos == iterPosCur`: We are at the key that has been returned.
+	// - `!skip && pos == iterPosCurForward`: We are at the key that has been returned.
 	//   To move forward we advance by one key, even if that lands us in the same
 	//   snapshot stripe.
-	// - `skip && pos == iterPosCur`: We are at the key that has been returned.
+	// - `skip && pos == iterPosCurForward`: We are at the key that has been returned.
 	//   To move forward we skip skippable entries in the stripe.
+	// - `skip && pos == iterPosNext && i.iterStripeChange == sameStripeNonSkippable`:
+	//    This case may occur when skipping within a snapshot stripe and we
+	//    encounter either:
+	//      a) an invalid key kind; The previous call will have returned
+	//         whatever key it was processing and deferred handling of the
+	//         invalid key to this invocation of Next(). We're responsible for
+	//         ignoring skip=true and falling into the invalid key kind case
+	//         down below.
+	//      b) an interleaved range delete; This is a wart of the current code
+	//         structure. While skipping within a snapshot stripe, a range
+	//         delete interleaved at its start key and sequence number
+	//         interrupts the sequence of point keys. After we return the range
+	//         delete to the caller, we need to pick up skipping at where we
+	//         left off, so we preserve skip=true.
+	//    TODO(jackson): This last case is confusing and can be removed if we
+	//    interleave range deletions at the maximal sequence number using the
+	//    keyspan interleaving iterator. This is the treatment given to range
+	//    keys today.
 	if i.pos == iterPosCurForward {
 		if i.skip {
 			i.skipInStripe()
 		} else {
 			i.nextInStripe()
 		}
+	} else if i.skip {
+		if i.iterStripeChange != sameStripeNonSkippable {
+			panic(errors.AssertionFailedf("compaction iterator has skip=true, but iterator is at iterPosNext"))
+		}
 	}
 
 	i.pos = iterPosCurForward
@@ -374,6 +429,10 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			return &i.key, i.value
 		}
 
+		// TODO(sumeer): we could avoid calling Covers if i.iterStripeChange ==
+		// sameStripeSameKey since that check has already been done in
+		// nextInStripeHelper. However, we also need to handle the case of
+		// CoversInvisibly below.
 		if cover := i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum); cover == keyspan.CoversVisibly {
 			// A pending range deletion deletes this key. Skip it.
 			i.saveKey()
@@ -382,17 +441,37 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 		} else if cover == keyspan.CoversInvisibly {
 			// i.iterKey would be deleted by a range deletion if there weren't
 			// any open snapshots. Mark it as pinned.
+			//
+			// NB: there are multiple places in this file where we call
+			// i.rangeDelFrag.Covers and this is the only one where we are writing
+			// to i.snapshotPinned. Those other cases occur in mergeNext where the
+			// caller is deciding whether the value should be merged or not, and the
+			// key is in the same snapshot stripe. Hence, snapshotPinned is by
+			// definition false in those cases.
 			i.snapshotPinned = true
+			i.forceObsoleteDueToRangeDel = true
+		} else {
+			i.forceObsoleteDueToRangeDel = false
 		}
 
 		switch i.iterKey.Kind() {
-		case InternalKeyKindDelete, InternalKeyKindSingleDelete:
+		case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
 			if i.elideTombstone(i.iterKey.UserKey) {
 				if i.curSnapshotIdx == 0 {
 					// If we're at the last snapshot stripe and the tombstone
 					// can be elided skip skippable keys in the same stripe.
 					i.saveKey()
-					i.skipInStripe()
+					if i.key.Kind() == InternalKeyKindSingleDelete {
+						i.skipDueToSingleDeleteElision()
+					} else {
+						i.skipInStripe()
+						if !i.skip && i.iterStripeChange != newStripeNewKey {
+							panic(errors.AssertionFailedf("pebble: skipInStripe in last stripe disabled skip without advancing to new key"))
+						}
+					}
+					if i.iterStripeChange == newStripeSameKey {
+						panic(errors.AssertionFailedf("pebble: skipInStripe in last stripe found a new stripe within the same key"))
+					}
 					continue
 				} else {
 					// We're not at the last snapshot stripe, so the tombstone
@@ -410,11 +489,23 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 				i.skip = true
 				return &i.key, i.value
 
+			case InternalKeyKindDeleteSized:
+				// We may skip subsequent keys because of this tombstone. Scan
+				// ahead to see just how much data this tombstone drops and if
+				// the tombstone's value should be updated accordingly.
+				return i.deleteSizedNext()
+
 			case InternalKeyKindSingleDelete:
 				if i.singleDeleteNext() {
 					return &i.key, i.value
+				} else if i.err != nil {
+					return nil, nil
 				}
 				continue
+
+			default:
+				panic(errors.AssertionFailedf(
+					"unexpected kind %s", redact.SafeString(i.iterKey.Kind().String())))
 			}
 
 		case InternalKeyKindSet, InternalKeyKindSetWithDelete:
@@ -424,6 +515,9 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			// preserving the original value, and potentially mutating the key
 			// kind.
 			i.setNext()
+			if i.err != nil {
+				return nil, nil
+			}
 			return &i.key, i.value
 
 		case InternalKeyKindMerge:
@@ -440,7 +534,15 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			if i.err == nil {
 				// includesBase is true whenever we've transformed the MERGE record
 				// into a SET.
-				includesBase := i.key.Kind() == InternalKeyKindSet
+				var includesBase bool
+				switch i.key.Kind() {
+				case InternalKeyKindSet, InternalKeyKindSetWithDelete:
+					includesBase = true
+				case InternalKeyKindMerge:
+				default:
+					panic(errors.AssertionFailedf(
+						"unexpected kind %s", redact.SafeString(i.key.Kind().String())))
+				}
 				i.value, needDelete, i.valueCloser, i.err = finishValueMerger(valueMerger, includesBase)
 			}
 			if i.err == nil {
@@ -469,6 +571,8 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			}
 			if i.err != nil {
 				i.valid = false
+				// TODO(sumeer): why is MarkCorruptionError only being called for
+				// MERGE?
 				i.err = base.MarkCorruptionError(i.err)
 			}
 			return nil, nil
@@ -508,10 +612,16 @@ func snapshotIndex(seq uint64, snapshots []uint64) (int, uint64) {
 	return index, snapshots[index]
 }
 
-// skipInStripe skips over skippable keys in the same stripe and user key.
+// skipInStripe skips over skippable keys in the same stripe and user key. It
+// may set i.err, in which case i.iterKey will be nil.
 func (i *compactionIter) skipInStripe() {
 	i.skip = true
+	// TODO(sumeer): we can avoid the overhead of calling i.rangeDelFrag.Covers,
+	// in this case of nextInStripe, since we are skipping all of them anyway.
 	for i.nextInStripe() == sameStripeSkippable {
+		if i.err != nil {
+			panic(i.err)
+		}
 	}
 	// Reset skip if we landed outside the original stripe. Otherwise, we landed
 	// in the same stripe on a non-skippable key. In that case we should preserve
@@ -549,11 +659,17 @@ const (
 // nextInStripe advances the iterator and returns one of the above const ints
 // indicating how its state changed.
 //
+// All sameStripeSkippable keys that are covered by a RANGEDEL will be skipped
+// and not returned.
+//
 // Calls to nextInStripe must be preceded by a call to saveKey to retain a
 // temporary reference to the original key, so that forward iteration can
 // proceed with a reference to the original key. Care should be taken to avoid
 // overwriting or mutating the saved key or value before they have been returned
 // to the caller of the exported function (i.e. the caller of Next, First, etc.)
+//
+// nextInStripe may set i.err, in which case the return value will be
+// newStripeNewKey, and i.iterKey will be nil.
 func (i *compactionIter) nextInStripe() stripeChangeType {
 	i.iterStripeChange = i.nextInStripeHelper()
 	return i.iterStripeChange
@@ -562,58 +678,72 @@ func (i *compactionIter) nextInStripe() stripeChangeType {
 // nextInStripeHelper is an internal helper for nextInStripe; callers should use
 // nextInStripe and not call nextInStripeHelper.
 func (i *compactionIter) nextInStripeHelper() stripeChangeType {
-	if !i.iterNext() {
+	origSnapshotIdx := i.curSnapshotIdx
-		return newStripeNewKey
+	for {
-	}
+		if !i.iterNext() {
-	key := i.iterKey
+			return newStripeNewKey
+		}
+		key := i.iterKey
 
-	// NB: The below conditional is an optimization to avoid a user key
+		if !i.equal(i.key.UserKey, key.UserKey) {
-	// comparison in many cases. Internal keys with the same user key are
+			i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
-	// ordered in (strictly) descending order by trailer. If the new key has a
+			return newStripeNewKey
-	// greater or equal trailer, or the previous key had a zero sequence number,
+		}
-	// the new key must have a new user key.
+
-	//
+		// If i.key and key have the same user key, then
-	// A couple things make these cases common:
+		//   1. i.key must not have had a zero sequence number (or it would've be the last
-	// - Sequence-number zeroing ensures ~all of the keys in L6 have a zero
+		//      key with its user key).
-	//   sequence number.
+		//   2. i.key must have a strictly larger sequence number
-	// - Ingested sstables' keys all adopt the same sequence number.
+		// There's an exception in that either key may be a range delete. Range
-	if i.keyTrailer <= base.InternalKeyZeroSeqnumMaxTrailer || key.Trailer >= i.keyTrailer {
+		// deletes may share a sequence number with a point key if the keys were
-		if invariants.Enabled && i.equal(i.key.UserKey, key.UserKey) {
+		// ingested together. Range keys may also share the sequence number if they
+		// were ingested, but range keys are interleaved into the compaction
+		// iterator's input iterator at the maximal sequence number so their
+		// original sequence number will not be observed here.
+		if prevSeqNum := base.SeqNumFromTrailer(i.keyTrailer); (prevSeqNum == 0 || prevSeqNum <= key.SeqNum()) &&
+			i.key.Kind() != InternalKeyKindRangeDelete && key.Kind() != InternalKeyKindRangeDelete {
 			prevKey := i.key
 			prevKey.Trailer = i.keyTrailer
-			panic(fmt.Sprintf("pebble: invariant violation: %s and %s out of order", key, prevKey))
+			panic(errors.AssertionFailedf("pebble: invariant violation: %s and %s out of order", prevKey, key))
 		}
+
 		i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
-		return newStripeNewKey
+		switch key.Kind() {
-	} else if !i.equal(i.key.UserKey, key.UserKey) {
+		case InternalKeyKindRangeDelete:
-		i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
+			// Range tombstones need to be exposed by the compactionIter to the upper level
-		return newStripeNewKey
+			// `compaction` object, so return them regardless of whether they are in the same
-	}
+			// snapshot stripe.
-	origSnapshotIdx := i.curSnapshotIdx
+			if i.curSnapshotIdx == origSnapshotIdx {
-	i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
+				return sameStripeNonSkippable
-	switch key.Kind() {
+			}
-	case InternalKeyKindRangeDelete:
+			return newStripeSameKey
-		// Range tombstones need to be exposed by the compactionIter to the upper level
+		case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
-		// `compaction` object, so return them regardless of whether they are in the same
+			// Range keys are interleaved at the max sequence number for a given user
-		// snapshot stripe.
+			// key, so we should not see any more range keys in this stripe.
-		if i.curSnapshotIdx == origSnapshotIdx {
+			panic("unreachable")
-			return sameStripeNonSkippable
+		case InternalKeyKindInvalid:
+			if i.curSnapshotIdx == origSnapshotIdx {
+				return sameStripeNonSkippable
+			}
+			return newStripeSameKey
+		case InternalKeyKindDelete, InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSingleDelete,
+			InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
+			// Fall through
+		default:
+			i.iterKey = nil
+			i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
+			i.valid = false
+			return newStripeNewKey
 		}
-		return newStripeSameKey
-	case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
-		// Range keys are interleaved at the max sequence number for a given user
-		// key, so we should not see any more range keys in this stripe.
-		panic("unreachable")
-	case InternalKeyKindInvalid:
 		if i.curSnapshotIdx == origSnapshotIdx {
-			return sameStripeNonSkippable
+			// Same snapshot.
+			if i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
+				continue
+			}
+			return sameStripeSkippable
 		}
 		return newStripeSameKey
 	}
-	if i.curSnapshotIdx == origSnapshotIdx {
-		return sameStripeSkippable
-	}
-	return newStripeSameKey
 }
 
 func (i *compactionIter) setNext() {
@@ -639,6 +769,10 @@ func (i *compactionIter) setNext() {
 
 	// Else, we continue to loop through entries in the stripe looking for a
 	// DEL. Note that we may stop *before* encountering a DEL, if one exists.
+	//
+	// NB: nextInStripe will skip sameStripeSkippable keys that are visibly
+	// covered by a RANGEDEL. This can include DELs -- this is fine since such
+	// DELs don't need to be combined with SET to make SETWITHDEL.
 	for {
 		switch i.nextInStripe() {
 		case newStripeNewKey, newStripeSameKey:
@@ -686,14 +820,19 @@ func (i *compactionIter) setNext() {
 			i.skip = true
 			return
 		case sameStripeSkippable:
-			// We're still in the same stripe. If this is a DEL/SINGLEDEL, we
+			// We're still in the same stripe. If this is a
-			// stop looking and emit a SETWITHDEL. Subsequent keys are
+			// DEL/SINGLEDEL/DELSIZED, we stop looking and emit a SETWITHDEL.
-			// eligible for skipping.
+			// Subsequent keys are eligible for skipping.
-			if i.iterKey.Kind() == InternalKeyKindDelete ||
+			switch i.iterKey.Kind() {
-				i.iterKey.Kind() == InternalKeyKindSingleDelete {
+			case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
 				i.key.SetKind(InternalKeyKindSetWithDelete)
 				i.skip = true
 				return
+			case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
+				// Do nothing
+			default:
+				i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
+				i.valid = false
 			}
 		default:
 			panic("pebble: unexpected stripeChangeType: " + strconv.Itoa(int(i.iterStripeChange)))
@@ -713,13 +852,24 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
 			i.pos = iterPosNext
 			return i.iterStripeChange
 		}
+		if i.err != nil {
+			panic(i.err)
+		}
+		// NB: MERGE#10+RANGEDEL#9 stays a MERGE, since nextInStripe skips
+		// sameStripeSkippable keys that are visibly covered by a RANGEDEL. There
+		// may be MERGE#7 that is invisibly covered and will be preserved, but
+		// there is no risk that MERGE#10 and MERGE#7 will get merged in the
+		// future as the RANGEDEL still exists and will be used in user-facing
+		// reads that see MERGE#10, and will also eventually cause MERGE#7 to be
+		// deleted in a compaction.
 		key := i.iterKey
 		switch key.Kind() {
-		case InternalKeyKindDelete, InternalKeyKindSingleDelete:
+		case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
 			// We've hit a deletion tombstone. Return everything up to this point and
 			// then skip entries until the next snapshot stripe. We change the kind
 			// of the result key to a Set so that it shadows keys in lower
-			// levels. That is, MERGE+DEL -> SET.
+			// levels. That is, MERGE+DEL -> SETWITHDEL.
+			//
 			// We do the same for SingleDelete since SingleDelete is only
 			// permitted (with deterministic behavior) for keys that have been
 			// set once since the last SingleDelete/Delete, so everything
@@ -732,21 +882,11 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
 			// single Set, and then merge in any following Sets, but that is
 			// complicated wrt code and unnecessary given the narrow permitted
 			// use of SingleDelete.
-			i.key.SetKind(InternalKeyKindSet)
+			i.key.SetKind(InternalKeyKindSetWithDelete)
 			i.skip = true
 			return sameStripeSkippable
 
 		case InternalKeyKindSet, InternalKeyKindSetWithDelete:
-			if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
-				// We change the kind of the result key to a Set so that it shadows
-				// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
-				// strictly necessary, but provides consistency with the behavior of
-				// MERGE+DEL.
-				i.key.SetKind(InternalKeyKindSet)
-				i.skip = true
-				return sameStripeSkippable
-			}
-
 			// We've hit a Set or SetWithDel value. Merge with the existing
 			// value and return. We change the kind of the resulting key to a
 			// Set so that it shadows keys in lower levels. That is:
@@ -761,16 +901,6 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
 			return sameStripeSkippable
 
 		case InternalKeyKindMerge:
-			if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
-				// We change the kind of the result key to a Set so that it shadows
-				// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
-				// strictly necessary, but provides consistency with the behavior of
-				// MERGE+DEL.
-				i.key.SetKind(InternalKeyKindSet)
-				i.skip = true
-				return sameStripeSkippable
-			}
-
 			// We've hit another Merge value. Merge with the existing value and
 			// continue looping.
 			i.err = valueMerger.MergeOlder(i.iterValue)
@@ -787,6 +917,16 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
 	}
 }
 
+// singleDeleteNext processes a SingleDelete point tombstone. A SingleDelete, or
+// SINGLEDEL, is unique in that it deletes exactly 1 internal key. It's a
+// performance optimization when the client knows a user key has not been
+// overwritten, allowing the elision of the tombstone earlier, avoiding write
+// amplification.
+//
+// singleDeleteNext returns a boolean indicating whether or not the caller
+// should yield the SingleDelete key to the consumer of the compactionIter. If
+// singleDeleteNext returns false, the caller may consume/elide the
+// SingleDelete.
 func (i *compactionIter) singleDeleteNext() bool {
 	// Save the current key.
 	i.saveKey()
@@ -795,26 +935,85 @@ func (i *compactionIter) singleDeleteNext() bool {
 
 	// Loop until finds a key to be passed to the next level.
 	for {
+		// If we find a key that can't be skipped, return true so that the
+		// caller yields the SingleDelete to the caller.
 		if i.nextInStripe() != sameStripeSkippable {
+			// This defers additional error checking regarding single delete
+			// invariants to the compaction where the keys with the same user key as
+			// the single delete are in the same stripe.
 			i.pos = iterPosNext
-			return true
+			return i.err == nil
 		}
-
+		if i.err != nil {
+			panic(i.err)
+		}
+		// INVARIANT: sameStripeSkippable.
 		key := i.iterKey
-		switch key.Kind() {
+		kind := key.Kind()
-		case InternalKeyKindDelete, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
+		switch kind {
-			// We've hit a Delete, Merge or SetWithDelete, transform the
+		case InternalKeyKindDelete, InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
-			// SingleDelete into a full Delete.
+			if (kind == InternalKeyKindDelete || kind == InternalKeyKindDeleteSized) &&
+				i.ineffectualSingleDeleteCallback != nil {
+				i.ineffectualSingleDeleteCallback(i.key.UserKey)
+			}
+			// We've hit a Delete, DeleteSized, SetWithDelete, transform
+			// the SingleDelete into a full Delete.
 			i.key.SetKind(InternalKeyKindDelete)
 			i.skip = true
 			return true
 
-		case InternalKeyKindSet:
+		case InternalKeyKindSet, InternalKeyKindMerge:
-			i.nextInStripe()
+			// This SingleDelete deletes the Set/Merge, and we can now elide the
+			// SingleDel as well. We advance past the Set and return false to
+			// indicate to the main compaction loop that we should NOT yield the
+			// current SingleDel key to the compaction loop.
+			//
+			// NB: singleDeleteNext was called with i.pos == iterPosCurForward, and
+			// after the call to nextInStripe, we are still at iterPosCurForward,
+			// since we are at the key after the Set/Merge that was single deleted.
+			change := i.nextInStripe()
+			switch change {
+			case sameStripeSkippable, newStripeSameKey:
+				// On the same user key.
+				nextKind := i.iterKey.Kind()
+				switch nextKind {
+				case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
+					if i.singleDeleteInvariantViolationCallback != nil {
+						// sameStripeSkippable keys returned by nextInStripe() are already
+						// known to not be covered by a RANGEDEL, so it is an invariant
+						// violation. The rare case is newStripeSameKey, where it is a
+						// violation if not covered by a RANGEDEL.
+						if change == sameStripeSkippable ||
+							i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) == keyspan.NoCover {
+							i.singleDeleteInvariantViolationCallback(i.key.UserKey)
+						}
+					}
+				case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete,
+					InternalKeyKindRangeDelete:
+				default:
+					panic(errors.AssertionFailedf(
+						"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
+				}
+			case sameStripeNonSkippable:
+				// No ability to check whether there is another Set/Merge below with
+				// the same user key.
+				//
+				// TODO(sumeer): once range deletions are interleaved at the maximal
+				// sequence number, this case will go away.
+			case newStripeNewKey:
+			default:
+				panic("unreachable")
+			}
 			i.valid = false
 			return false
 
 		case InternalKeyKindSingleDelete:
+			// Two single deletes met in a compaction. The first single delete is
+			// ineffectual.
+			if i.ineffectualSingleDeleteCallback != nil {
+				i.ineffectualSingleDeleteCallback(i.key.UserKey)
+			}
+			// Continue to apply the second single delete.
 			continue
 
 		default:
@@ -825,6 +1024,303 @@ func (i *compactionIter) singleDeleteNext() bool {
 	}
 }
 
+// skipDueToSingleDeleteElision is called when the SingleDelete is being
+// elided because it is in the final snapshot stripe and there are no keys
+// with the same user key in lower levels in the LSM (below the files in this
+// compaction).
+//
+// TODO(sumeer): the only difference between singleDeleteNext and
+// skipDueToSingleDeleteElision is the fact that the caller knows it will be
+// eliding the single delete in the latter case. There are some similar things
+// happening in both implementations. My first attempt at combining them into
+// a single method was hard to comprehend. Try again.
+func (i *compactionIter) skipDueToSingleDeleteElision() {
+	for {
+		stripeChange := i.nextInStripe()
+		if i.err != nil {
+			panic(i.err)
+		}
+		switch stripeChange {
+		case newStripeNewKey:
+			// The single delete is only now being elided, meaning it did not elide
+			// any keys earlier in its descent down the LSM. We stepped onto a new
+			// user key, meaning that even now at its moment of elision, it still
+			// hasn't elided any other keys. The single delete was ineffectual (a
+			// no-op).
+			if i.ineffectualSingleDeleteCallback != nil {
+				i.ineffectualSingleDeleteCallback(i.key.UserKey)
+			}
+			i.skip = false
+			return
+		case newStripeSameKey:
+			// This should be impossible. If we're eliding a single delete, we
+			// determined that the tombstone is in the final snapshot stripe, but we
+			// stepped into a new stripe of the same key.
+			panic(errors.AssertionFailedf("eliding single delete followed by same key in new stripe"))
+		case sameStripeNonSkippable:
+			// There's a key that we cannot skip. There are two possible cases:
+			//   a. The key is invalid. This is an error.
+			//   b. The key is a range deletion.
+			// The second case may also be an ineffectual single delete. However, it
+			// is possible that there is a SET that is at the same seqnum as the
+			// RANGEDEL, and so is not deleted by that RANGEDEL, and will be deleted
+			// by this single delete. So we cannot be certain that this is an
+			// ineffectual single delete.
+			//
+			// TODO(sumeer): the existing todo to interleave range deletions at the
+			// maximal sequence number will allow us to address this ambiguity.
+			//
+			// TODO(sumeer): by setting skip to true, the compactionIter is making a
+			// single delete stronger (like a del), which will hide bugs in the use of
+			// single delete.
+			i.skip = true
+			return
+		case sameStripeSkippable:
+			kind := i.iterKey.Kind()
+			switch kind {
+			case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete:
+				if i.ineffectualSingleDeleteCallback != nil {
+					i.ineffectualSingleDeleteCallback(i.key.UserKey)
+				}
+				switch kind {
+				case InternalKeyKindDelete, InternalKeyKindDeleteSized:
+					i.skipInStripe()
+					return
+				case InternalKeyKindSingleDelete:
+					// Repeat the same with this SingleDelete. We don't want to simply
+					// call skipInStripe(), since it increases the strength of the
+					// SingleDel, which hides bugs in the use of single delete.
+					continue
+				default:
+					panic(errors.AssertionFailedf(
+						"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
+				}
+			case InternalKeyKindSetWithDelete:
+				// The SingleDelete should behave like a Delete.
+				i.skipInStripe()
+				return
+			case InternalKeyKindSet, InternalKeyKindMerge:
+				// This SingleDelete deletes the Set/Merge, and we are eliding the
+				// SingleDel as well. Step to the next key (this is not deleted by the
+				// SingleDelete).
+				//
+				// NB: skipDueToSingleDeleteElision was called with i.pos ==
+				// iterPosCurForward, and after the call to nextInStripe, we are still
+				// at iterPosCurForward, since we are at the key after the Set/Merge
+				// that was single deleted.
+				change := i.nextInStripe()
+				if i.err != nil {
+					panic(i.err)
+				}
+				switch change {
+				case newStripeSameKey:
+					panic(errors.AssertionFailedf("eliding single delete followed by same key in new stripe"))
+				case newStripeNewKey:
+				case sameStripeSkippable:
+					// On the same key.
+					nextKind := i.iterKey.Kind()
+					switch nextKind {
+					case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
+						if i.singleDeleteInvariantViolationCallback != nil {
+							i.singleDeleteInvariantViolationCallback(i.key.UserKey)
+						}
+					case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete,
+						InternalKeyKindRangeDelete:
+					default:
+						panic(errors.AssertionFailedf(
+							"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
+					}
+				case sameStripeNonSkippable:
+					// No ability to check whether there is another Set/Merge below with
+					// the same user key.
+					//
+					// TODO(sumeer): once range deletions are interleaved at the maximal
+					// sequence number, this case will go away.
+				default:
+					panic("unreachable")
+				}
+				// Whether in same stripe or new stripe, this key is not consumed by
+				// the SingleDelete.
+				i.skip = false
+				return
+			default:
+				panic(errors.AssertionFailedf(
+					"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
+			}
+		default:
+			panic("unreachable")
+		}
+	}
+}
+
+// deleteSizedNext processes a DELSIZED point tombstone. Unlike ordinary DELs,
+// these tombstones carry a value that's a varint indicating the size of the
+// entry (len(key)+len(value)) that the tombstone is expected to delete.
+//
+// When a deleteSizedNext is encountered, we skip ahead to see which keys, if
+// any, are elided as a result of the tombstone.
+func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) {
+	i.saveKey()
+	i.valid = true
+	i.skip = true
+
+	// The DELSIZED tombstone may have no value at all. This happens when the
+	// tombstone has already deleted the key that the user originally predicted.
+	// In this case, we still peek forward in case there's another DELSIZED key
+	// with a lower sequence number, in which case we'll adopt its value.
+	if len(i.iterValue) == 0 {
+		i.value = i.valueBuf[:0]
+	} else {
+		i.valueBuf = append(i.valueBuf[:0], i.iterValue...)
+		i.value = i.valueBuf
+	}
+
+	// Loop through all the keys within this stripe that are skippable.
+	i.pos = iterPosNext
+	for i.nextInStripe() == sameStripeSkippable {
+		if i.err != nil {
+			panic(i.err)
+		}
+		switch i.iterKey.Kind() {
+		case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete:
+			// We encountered a tombstone (DEL, or DELSIZED) that's deleted by
+			// the original DELSIZED tombstone. This can happen in two cases:
+			//
+			// (1) These tombstones were intended to delete two distinct values,
+			//     and this DELSIZED has already dropped the relevant key. For
+			//     example:
+			//
+			//     a.DELSIZED.9   a.SET.7   a.DELSIZED.5   a.SET.4
+			//
+			//     If a.DELSIZED.9 has already deleted a.SET.7, its size has
+			//     already been zeroed out. In this case, we want to adopt the
+			//     value of the DELSIZED with the lower sequence number, in
+			//     case the a.SET.4 key has not yet been elided.
+			//
+			// (2) This DELSIZED was missized. The user thought they were
+			//     deleting a key with this user key, but this user key had
+			//     already been deleted.
+			//
+			// We can differentiate these two cases by examining the length of
+			// the DELSIZED's value. A DELSIZED's value holds the size of both
+			// the user key and value that it intends to delete. For any user
+			// key with a length > 0, a DELSIZED that has not deleted a key must
+			// have a value with a length > 0.
+			//
+			// We treat both cases the same functionally, adopting the identity
+			// of the lower-sequence numbered tombstone. However in the second
+			// case, we also increment the stat counting missized tombstones.
+			if len(i.value) > 0 {
+				// The original DELSIZED key was missized. The key that the user
+				// thought they were deleting does not exist.
+				i.stats.countMissizedDels++
+			}
+			i.valueBuf = append(i.valueBuf[:0], i.iterValue...)
+			i.value = i.valueBuf
+			if i.iterKey.Kind() != InternalKeyKindDeleteSized {
+				// Convert the DELSIZED to a DEL—The DEL/SINGLEDEL we're eliding
+				// may not have deleted the key(s) it was intended to yet. The
+				// ordinary DEL compaction heuristics are better suited at that,
+				// plus we don't want to count it as a missized DEL. We early
+				// exit in this case, after skipping the remainder of the
+				// snapshot stripe.
+				i.key.SetKind(InternalKeyKindDelete)
+				// NB: We skipInStripe now, rather than returning leaving
+				// i.skip=true and returning early, because Next() requires
+				// that i.skip=true only if i.iterPos = iterPosCurForward.
+				//
+				// Ignore any error caused by skipInStripe since it does not affect
+				// the key/value being returned here, and the next call to Next() will
+				// expose it.
+				i.skipInStripe()
+				return &i.key, i.value
+			}
+			// Continue, in case we uncover another DELSIZED or a key this
+			// DELSIZED deletes.
+
+		case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
+			// If the DELSIZED is value-less, it already deleted the key that it
+			// was intended to delete. This is possible with a sequence like:
+			//
+			//      DELSIZED.8     SET.7     SET.3
+			//
+			// The DELSIZED only describes the size of the SET.7, which in this
+			// case has already been elided. We don't count it as a missizing,
+			// instead converting the DELSIZED to a DEL. Skip the remainder of
+			// the snapshot stripe and return.
+			if len(i.value) == 0 {
+				i.key.SetKind(InternalKeyKindDelete)
+				// NB: We skipInStripe now, rather than returning leaving
+				// i.skip=true and returning early, because Next() requires
+				// that i.skip=true only if i.iterPos = iterPosCurForward.
+				//
+				// Ignore any error caused by skipInStripe since it does not affect
+				// the key/value being returned here, and the next call to Next() will
+				// expose it.
+				i.skipInStripe()
+				return &i.key, i.value
+			}
+			// The deleted key is not a DEL, DELSIZED, and the DELSIZED in i.key
+			// has a positive size.
+			expectedSize, n := binary.Uvarint(i.value)
+			if n != len(i.value) {
+				i.err = base.CorruptionErrorf("DELSIZED holds invalid value: %x", errors.Safe(i.value))
+				i.valid = false
+				return nil, nil
+			}
+			elidedSize := uint64(len(i.iterKey.UserKey)) + uint64(len(i.iterValue))
+			if elidedSize != expectedSize {
+				// The original DELSIZED key was missized. It's unclear what to
+				// do. The user-provided size was wrong, so it's unlikely to be
+				// accurate or meaningful. We could:
+				//
+				//   1. return the DELSIZED with the original user-provided size unmodified
+				//   2. return the DELZIZED with a zeroed size to reflect that a key was
+				//   elided, even if it wasn't the anticipated size.
+				//   3. subtract the elided size from the estimate and re-encode.
+				//   4. convert the DELSIZED into a value-less DEL, so that
+				//      ordinary DEL heuristics apply.
+				//
+				// We opt for (4) under the rationale that we can't rely on the
+				// user-provided size for accuracy, so ordinary DEL heuristics
+				// are safer.
+				i.stats.countMissizedDels++
+				i.key.SetKind(InternalKeyKindDelete)
+				i.value = i.valueBuf[:0]
+				// NB: We skipInStripe now, rather than returning leaving
+				// i.skip=true and returning early, because Next() requires
+				// that i.skip=true only if i.iterPos = iterPosCurForward.
+				//
+				// Ignore any error caused by skipInStripe since it does not affect
+				// the key/value being returned here, and the next call to Next() will
+				// expose it.
+				i.skipInStripe()
+				return &i.key, i.value
+			}
+			// NB: We remove the value regardless of whether the key was sized
+			// appropriately. The size encoded is 'consumed' the first time it
+			// meets a key that it deletes.
+			i.value = i.valueBuf[:0]
+
+		default:
+			i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
+			i.valid = false
+			return nil, nil
+		}
+	}
+	// Reset skip if we landed outside the original stripe. Otherwise, we landed
+	// in the same stripe on a non-skippable key. In that case we should preserve
+	// `i.skip == true` such that later keys in the stripe will continue to be
+	// skipped.
+	if i.iterStripeChange == newStripeNewKey || i.iterStripeChange == newStripeSameKey {
+		i.skip = false
+	}
+	if i.err != nil {
+		return nil, nil
+	}
+	return &i.key, i.value
+}
+
 func (i *compactionIter) saveKey() {
 	i.keyBuf = append(i.keyBuf[:0], i.iterKey.UserKey...)
 	i.key.UserKey = i.keyBuf
@@ -957,7 +1453,7 @@ func (i *compactionIter) maybeZeroSeqnum(snapshotIdx int) {
 		// This is not the last snapshot
 		return
 	}
-	i.key.SetSeqNum(0)
+	i.key.SetSeqNum(base.SeqNumZero)
 }
 
 // A frontier is used to monitor a compaction's progression across the user

vendor/github.com/cockroachdb/pebble/event.go

@@ -39,10 +39,11 @@ func formatFileNums(tables []TableInfo) string {
 	return buf.String()
 }
 
-// LevelInfo contains info pertaining to a partificular level.
+// LevelInfo contains info pertaining to a particular level.
 type LevelInfo struct {
 	Level  int
 	Tables []TableInfo
+	Score  float64
 }
 
 func (i LevelInfo) String() string {
@@ -51,8 +52,11 @@ func (i LevelInfo) String() string {
 
 // SafeFormat implements redact.SafeFormatter.
 func (i LevelInfo) SafeFormat(w redact.SafePrinter, _ rune) {
-	w.Printf("L%d [%s] (%s)", redact.Safe(i.Level), redact.Safe(formatFileNums(i.Tables)),
+	w.Printf("L%d [%s] (%s) Score=%.2f",
-		redact.Safe(humanize.Uint64(tablesTotalSize(i.Tables))))
+		redact.Safe(i.Level),
+		redact.Safe(formatFileNums(i.Tables)),
+		redact.Safe(humanize.Bytes.Uint64(tablesTotalSize(i.Tables))),
+		redact.Safe(i.Score))
 }
 
 // CompactionInfo contains the info for a compaction event.
@@ -75,6 +79,27 @@ type CompactionInfo struct {
 	TotalDuration time.Duration
 	Done          bool
 	Err           error
+
+	SingleLevelOverlappingRatio float64
+	MultiLevelOverlappingRatio  float64
+
+	// Annotations specifies additional info to appear in a compaction's event log line
+	Annotations compactionAnnotations
+}
+
+type compactionAnnotations []string
+
+// SafeFormat implements redact.SafeFormatter.
+func (ca compactionAnnotations) SafeFormat(w redact.SafePrinter, _ rune) {
+	if len(ca) == 0 {
+		return
+	}
+	for i := range ca {
+		if i != 0 {
+			w.Print(" ")
+		}
+		w.Printf("%s", redact.SafeString(ca[i]))
+	}
 }
 
 func (i CompactionInfo) String() string {
@@ -90,20 +115,25 @@ func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 	}
 
 	if !i.Done {
-		w.Printf("[JOB %d] compacting(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
+		w.Printf("[JOB %d] compacting(%s) ",
-		w.Print(levelInfos(i.Input))
+			redact.Safe(i.JobID),
+			redact.SafeString(i.Reason))
+		w.Printf("%s", i.Annotations)
+		w.Printf("%s; ", levelInfos(i.Input))
+		w.Printf("OverlappingRatio: Single %.2f, Multi %.2f", i.SingleLevelOverlappingRatio, i.MultiLevelOverlappingRatio)
 		return
 	}
 	outputSize := tablesTotalSize(i.Output.Tables)
 	w.Printf("[JOB %d] compacted(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
+	w.Printf("%s", i.Annotations)
 	w.Print(levelInfos(i.Input))
 	w.Printf(" -> L%d [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
 		redact.Safe(i.Output.Level),
 		redact.Safe(formatFileNums(i.Output.Tables)),
-		redact.Safe(humanize.Uint64(outputSize)),
+		redact.Safe(humanize.Bytes.Uint64(outputSize)),
 		redact.Safe(i.Duration.Seconds()),
 		redact.Safe(i.TotalDuration.Seconds()),
-		redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
+		redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
 }
 
 type levelInfos []LevelInfo
@@ -129,6 +159,9 @@ type FlushInfo struct {
 	Reason string
 	// Input contains the count of input memtables that were flushed.
 	Input int
+	// InputBytes contains the total in-memory size of the memtable(s) that were
+	// flushed. This size includes skiplist indexing data structures.
+	InputBytes uint64
 	// Output contains the ouptut table generated by the flush. The output info
 	// is empty for the flush begin event.
 	Output []TableInfo
@@ -168,7 +201,7 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		if !i.Ingest {
 			w.Printf("flushing %d memtable", redact.Safe(i.Input))
 			w.SafeString(plural)
-			w.Printf(" to L0")
+			w.Printf(" (%s) to L0", redact.Safe(humanize.Bytes.Uint64(i.InputBytes)))
 		} else {
 			w.Printf("flushing %d ingested table%s", redact.Safe(i.Input), plural)
 		}
@@ -180,13 +213,14 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		if invariants.Enabled && len(i.IngestLevels) > 0 {
 			panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) == 0"))
 		}
-		w.Printf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
+		w.Printf("[JOB %d] flushed %d memtable%s (%s) to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
 			redact.Safe(i.JobID), redact.Safe(i.Input), plural,
+			redact.Safe(humanize.Bytes.Uint64(i.InputBytes)),
 			redact.Safe(formatFileNums(i.Output)),
-			redact.Safe(humanize.Uint64(outputSize)),
+			redact.Safe(humanize.Bytes.Uint64(outputSize)),
 			redact.Safe(i.Duration.Seconds()),
 			redact.Safe(i.TotalDuration.Seconds()),
-			redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
+			redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
 	} else {
 		if invariants.Enabled && len(i.IngestLevels) == 0 {
 			panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) > 0"))
@@ -198,12 +232,12 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 			if j > 0 {
 				w.Printf(" +")
 			}
-			w.Printf(" L%d:%s (%s)", level, redact.Safe(file.FileNum), humanize.IEC.Uint64(file.Size))
+			w.Printf(" L%d:%s (%s)", level, file.FileNum, humanize.Bytes.Uint64(file.Size))
 		}
 		w.Printf(" in %.1fs (%.1fs total), output rate %s/s",
 			redact.Safe(i.Duration.Seconds()),
 			redact.Safe(i.TotalDuration.Seconds()),
-			redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
+			redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
 	}
 }
 
@@ -227,7 +261,7 @@ func (i ManifestCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		w.Printf("[JOB %d] MANIFEST create error: %s", redact.Safe(i.JobID), i.Err)
 		return
 	}
-	w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+	w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), i.FileNum)
 }
 
 // ManifestDeleteInfo contains the info for a Manifest deletion event.
@@ -249,7 +283,7 @@ func (i ManifestDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		w.Printf("[JOB %d] MANIFEST delete error: %s", redact.Safe(i.JobID), i.Err)
 		return
 	}
-	w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+	w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), i.FileNum)
 }
 
 // TableCreateInfo contains the info for a table creation event.
@@ -269,7 +303,7 @@ func (i TableCreateInfo) String() string {
 // SafeFormat implements redact.SafeFormatter.
 func (i TableCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 	w.Printf("[JOB %d] %s: sstable created %s",
-		redact.Safe(i.JobID), redact.Safe(i.Reason), redact.Safe(i.FileNum))
+		redact.Safe(i.JobID), redact.Safe(i.Reason), i.FileNum)
 }
 
 // TableDeleteInfo contains the info for a table deletion event.
@@ -288,10 +322,10 @@ func (i TableDeleteInfo) String() string {
 func (i TableDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 	if i.Err != nil {
 		w.Printf("[JOB %d] sstable delete error %s: %s",
-			redact.Safe(i.JobID), redact.Safe(i.FileNum), i.Err)
+			redact.Safe(i.JobID), i.FileNum, i.Err)
 		return
 	}
-	w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+	w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), i.FileNum)
 }
 
 // TableIngestInfo contains the info for a table ingestion event.
@@ -337,8 +371,8 @@ func (i TableIngestInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		if !i.flushable {
 			levelStr = fmt.Sprintf("L%d:", t.Level)
 		}
-		w.Printf(" %s%s (%s)", redact.Safe(levelStr), redact.Safe(t.FileNum),
+		w.Printf(" %s%s (%s)", redact.Safe(levelStr), t.FileNum,
-			redact.Safe(humanize.Uint64(t.Size)))
+			redact.Safe(humanize.Bytes.Uint64(t.Size)))
 	}
 }
 
@@ -399,12 +433,12 @@ func (i WALCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 	}
 
 	if i.RecycledFileNum == 0 {
-		w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+		w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), i.FileNum)
 		return
 	}
 
 	w.Printf("[JOB %d] WAL created %s (recycled %s)",
-		redact.Safe(i.JobID), redact.Safe(i.FileNum), redact.Safe(i.RecycledFileNum))
+		redact.Safe(i.JobID), i.FileNum, i.RecycledFileNum)
 }
 
 // WALDeleteInfo contains the info for a WAL deletion event.
@@ -426,7 +460,7 @@ func (i WALDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		w.Printf("[JOB %d] WAL delete error: %s", redact.Safe(i.JobID), i.Err)
 		return
 	}
-	w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+	w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), i.FileNum)
 }
 
 // WriteStallBeginInfo contains the info for a write stall begin event.

vendor/github.com/cockroachdb/pebble/external_iterator.go

@@ -209,15 +209,15 @@ func createExternalPointIter(ctx context.Context, it *Iterator) (internalIterato
 				pointIter    internalIterator
 				err          error
 			)
-			pointIter, err = r.NewIterWithBlockPropertyFiltersAndContext(
+			// We could set hideObsoletePoints=true, since we are reading at
-				ctx,
+			// InternalKeySeqNumMax, but we don't bother since these sstables should
-				it.opts.LowerBound,
+			// not have obsolete points (so the performance optimization is
-				it.opts.UpperBound,
+			// unnecessary), and we don't want to bother constructing a
-				nil,   /* BlockPropertiesFilterer */
+			// BlockPropertiesFilterer that includes obsoleteKeyBlockPropertyFilter.
-				false, /* useFilterBlock */
+			pointIter, err = r.NewIterWithBlockPropertyFiltersAndContextEtc(
-				&it.stats.InternalStats,
+				ctx, it.opts.LowerBound, it.opts.UpperBound, nil, /* BlockPropertiesFilterer */
-				sstable.TrivialReaderProvider{Reader: r},
+				false /* hideObsoletePoints */, false, /* useFilterBlock */
-			)
+				&it.stats.InternalStats, sstable.TrivialReaderProvider{Reader: r})
 			if err != nil {
 				return nil, err
 			}
@@ -311,7 +311,7 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) {
 					base.InternalKeySeqNumMax,
 					it.opts.LowerBound, it.opts.UpperBound,
 					&it.hasPrefix, &it.prefixOrFullSeekKey,
-					true /* onlySets */, &it.rangeKey.internal,
+					false /* internalKeys */, &it.rangeKey.internal,
 				)
 				for i := range rangeKeyIters {
 					it.rangeKey.iterConfig.AddLevel(rangeKeyIters[i])
@@ -319,8 +319,12 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) {
 			}
 		}
 		if it.rangeKey != nil {
-			it.rangeKey.iiter.Init(&it.comparer, it.iter, it.rangeKey.rangeKeyIter, &it.rangeKeyMasking,
+			it.rangeKey.iiter.Init(&it.comparer, it.iter, it.rangeKey.rangeKeyIter,
-				it.opts.LowerBound, it.opts.UpperBound)
+				keyspan.InterleavingIterOpts{
+					Mask:       &it.rangeKeyMasking,
+					LowerBound: it.opts.LowerBound,
+					UpperBound: it.opts.UpperBound,
+				})
 			it.iter = &it.rangeKey.iiter
 		}
 	}

vendor/github.com/cockroachdb/pebble/filenames.go

@@ -33,7 +33,7 @@ const (
 // NB: This is a low-level routine and typically not what you want to
 // use. Newer versions of Pebble running newer format major versions do
 // not use the CURRENT file. See setCurrentFunc in version_set.go.
-func setCurrentFile(dirname string, fs vfs.FS, fileNum FileNum) error {
+func setCurrentFile(dirname string, fs vfs.FS, fileNum base.DiskFileNum) error {
 	newFilename := base.MakeFilepath(fs, dirname, fileTypeCurrent, fileNum)
 	oldFilename := base.MakeFilepath(fs, dirname, fileTypeTemp, fileNum)
 	fs.Remove(oldFilename)

vendor/github.com/cockroachdb/pebble/flushable.go

@@ -60,20 +60,20 @@ type flushableEntry struct {
 	// flushable is a memTable, when the reader refs drops to zero, the writer
 	// refs will already be zero because the memtable will have been flushed and
 	// that only occurs once the writer refs drops to zero.
-	readerRefs int32
+	readerRefs atomic.Int32
 	// Closure to invoke to release memory accounting.
 	releaseMemAccounting func()
 	// unrefFiles, if not nil, should be invoked to decrease the ref count of
 	// files which are backing the flushable.
-	unrefFiles func() []*fileMetadata
+	unrefFiles func() []*fileBacking
 	// deleteFnLocked should be called if the caller is holding DB.mu.
-	deleteFnLocked func(obsolete []*fileMetadata)
+	deleteFnLocked func(obsolete []*fileBacking)
 	// deleteFn should be called if the caller is not holding DB.mu.
-	deleteFn func(obsolete []*fileMetadata)
+	deleteFn func(obsolete []*fileBacking)
 }
 
 func (e *flushableEntry) readerRef() {
-	switch v := atomic.AddInt32(&e.readerRefs, 1); {
+	switch v := e.readerRefs.Add(1); {
 	case v <= 1:
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
 	}
@@ -90,9 +90,9 @@ func (e *flushableEntry) readerUnrefLocked(deleteFiles bool) {
 }
 
 func (e *flushableEntry) readerUnrefHelper(
-	deleteFiles bool, deleteFn func(obsolete []*manifest.FileMetadata),
+	deleteFiles bool, deleteFn func(obsolete []*fileBacking),
 ) {
-	switch v := atomic.AddInt32(&e.readerRefs, -1); {
+	switch v := e.readerRefs.Add(-1); {
 	case v < 0:
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
 	case v == 0:
@@ -116,9 +116,8 @@ type flushableList []*flushableEntry
 // ingestedFlushable is the implementation of the flushable interface for the
 // ingesting sstables which are added to the flushable list.
 type ingestedFlushable struct {
-	files            []*fileMetadata
+	files            []physicalMeta
-	cmp              Compare
+	comparer         *Comparer
-	split            Split
 	newIters         tableNewIters
 	newRangeKeyIters keyspan.TableNewSpanIter
 
@@ -131,26 +130,27 @@ type ingestedFlushable struct {
 
 func newIngestedFlushable(
 	files []*fileMetadata,
-	cmp Compare,
+	comparer *Comparer,
-	split Split,
 	newIters tableNewIters,
 	newRangeKeyIters keyspan.TableNewSpanIter,
 ) *ingestedFlushable {
+	var physicalFiles []physicalMeta
+	var hasRangeKeys bool
+	for _, f := range files {
+		if f.HasRangeKeys {
+			hasRangeKeys = true
+		}
+		physicalFiles = append(physicalFiles, f.PhysicalMeta())
+	}
+
 	ret := &ingestedFlushable{
-		files:            files,
+		files:            physicalFiles,
-		cmp:              cmp,
+		comparer:         comparer,
-		split:            split,
 		newIters:         newIters,
 		newRangeKeyIters: newRangeKeyIters,
 		// slice is immutable and can be set once and used many times.
-		slice: manifest.NewLevelSliceKeySorted(cmp, files),
+		slice:        manifest.NewLevelSliceKeySorted(comparer.Compare, files),
-	}
+		hasRangeKeys: hasRangeKeys,
-
-	for _, f := range files {
-		if f.HasRangeKeys {
-			ret.hasRangeKeys = true
-			break
-		}
 	}
 
 	return ret
@@ -159,6 +159,7 @@ func newIngestedFlushable(
 // TODO(sumeer): ingestedFlushable iters also need to plumb context for
 // tracing.
 
+// newIter is part of the flushable interface.
 func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator {
 	var opts IterOptions
 	if o != nil {
@@ -169,10 +170,11 @@ func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator {
 	// aren't truly levels in the lsm. Right now, the encoding only supports
 	// L0 sublevels, and the rest of the levels in the lsm.
 	return newLevelIter(
-		opts, s.cmp, s.split, s.newIters, s.slice.Iter(), manifest.Level(0), nil,
+		opts, s.comparer, s.newIters, s.slice.Iter(), manifest.Level(0), internalIterOpts{},
 	)
 }
 
+// newFlushIter is part of the flushable interface.
 func (s *ingestedFlushable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
 	// newFlushIter is only used for writing memtables to disk as sstables.
 	// Since ingested sstables are already present on disk, they don't need to
@@ -181,7 +183,7 @@ func (s *ingestedFlushable) newFlushIter(o *IterOptions, bytesFlushed *uint64) i
 }
 
 func (s *ingestedFlushable) constructRangeDelIter(
-	file *manifest.FileMetadata, _ *keyspan.SpanIterOptions,
+	file *manifest.FileMetadata, _ keyspan.SpanIterOptions,
 ) (keyspan.FragmentIterator, error) {
 	// Note that the keyspan level iter expects a non-nil iterator to be
 	// returned even if there is an error. So, we return the emptyKeyspanIter.
@@ -196,41 +198,47 @@ func (s *ingestedFlushable) constructRangeDelIter(
 	return rangeDelIter, nil
 }
 
+// newRangeDelIter is part of the flushable interface.
 // TODO(bananabrick): Using a level iter instead of a keyspan level iter to
 // surface range deletes is more efficient.
 func (s *ingestedFlushable) newRangeDelIter(_ *IterOptions) keyspan.FragmentIterator {
 	return keyspan.NewLevelIter(
-		keyspan.SpanIterOptions{}, s.cmp,
+		keyspan.SpanIterOptions{}, s.comparer.Compare,
 		s.constructRangeDelIter, s.slice.Iter(), manifest.Level(0),
 		manifest.KeyTypePoint,
 	)
 }
 
+// newRangeKeyIter is part of the flushable interface.
 func (s *ingestedFlushable) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator {
 	if !s.containsRangeKeys() {
 		return nil
 	}
 
 	return keyspan.NewLevelIter(
-		keyspan.SpanIterOptions{}, s.cmp, s.newRangeKeyIters,
+		keyspan.SpanIterOptions{}, s.comparer.Compare, s.newRangeKeyIters,
 		s.slice.Iter(), manifest.Level(0), manifest.KeyTypeRange,
 	)
 }
 
+// containsRangeKeys is part of the flushable interface.
 func (s *ingestedFlushable) containsRangeKeys() bool {
 	return s.hasRangeKeys
 }
 
+// inuseBytes is part of the flushable interface.
 func (s *ingestedFlushable) inuseBytes() uint64 {
 	// inuseBytes is only used when memtables are flushed to disk as sstables.
 	panic("pebble: not implemented")
 }
 
+// totalBytes is part of the flushable interface.
 func (s *ingestedFlushable) totalBytes() uint64 {
 	// We don't allocate additional bytes for the ingestedFlushable.
 	return 0
 }
 
+// readyForFlush is part of the flushable interface.
 func (s *ingestedFlushable) readyForFlush() bool {
 	// ingestedFlushable should always be ready to flush. However, note that
 	// memtables before the ingested sstables in the memtable queue must be

vendor/github.com/cockroachdb/pebble/format_major_version.go

@@ -28,10 +28,13 @@ import (
 // FormatVersion that the default corresponds to may change with time.
 type FormatMajorVersion uint64
 
+// SafeValue implements redact.SafeValue.
+func (v FormatMajorVersion) SafeValue() {}
+
 // String implements fmt.Stringer.
 func (v FormatMajorVersion) String() string {
-	// NB: This must not change. It's used as the value for the the
+	// NB: This must not change. It's used as the value for the on-disk
-	// on-disk version marker file.
+	// version marker file.
 	//
 	// Specifically, this value must always parse as a base 10 integer
 	// that fits in a uint64. We format it as zero-padded, 3-digit
@@ -110,12 +113,12 @@ const (
 
 	// 23.1 versions.
 
-	// FormatUnusedPrePebblev1MarkedCompacted is an unused format major version.
+	// formatUnusedPrePebblev1MarkedCompacted is an unused format major version.
 	// This format major version was originally intended to ship in the 23.1
 	// release. It was later decided that this should be deferred until a
 	// subsequent release. The original ordering is preserved so as not to
 	// introduce breaking changes in Cockroach.
-	FormatUnusedPrePebblev1MarkedCompacted
+	formatUnusedPrePebblev1MarkedCompacted
 
 	// FormatSSTableValueBlocks is a format major version that adds support for
 	// storing values in value blocks in the sstable. Value block support is not
@@ -151,8 +154,28 @@ const (
 	// compactions for files marked for compaction are complete.
 	FormatPrePebblev1MarkedCompacted
 
+	// FormatDeleteSizedAndObsolete is a format major version that adds support
+	// for deletion tombstones that encode the size of the value they're
+	// expected to delete. This format major version is required before the
+	// associated key kind may be committed through batch applications or
+	// ingests. It also adds support for keys that are marked obsolete (see
+	// sstable/format.go for details).
+	FormatDeleteSizedAndObsolete
+
+	// FormatVirtualSSTables is a format major version that adds support for
+	// virtual sstables that can reference a sub-range of keys in an underlying
+	// physical sstable. This information is persisted through new,
+	// backward-incompatible fields in the Manifest, and therefore requires
+	// a format major version.
+	FormatVirtualSSTables
+
+	// internalFormatNewest holds the newest format major version, including
+	// experimental ones excluded from the exported FormatNewest constant until
+	// they've stabilized. Used in tests.
+	internalFormatNewest FormatMajorVersion = iota - 1
+
 	// FormatNewest always contains the most recent format major version.
-	FormatNewest FormatMajorVersion = iota - 1
+	FormatNewest FormatMajorVersion = internalFormatNewest
 )
 
 // MaxTableFormat returns the maximum sstable.TableFormat that can be used at
@@ -166,11 +189,12 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
 		FormatSplitUserKeysMarkedCompacted:
 		return sstable.TableFormatPebblev1
 	case FormatRangeKeys, FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
-		FormatUnusedPrePebblev1MarkedCompacted:
+		formatUnusedPrePebblev1MarkedCompacted:
 		return sstable.TableFormatPebblev2
-	case FormatSSTableValueBlocks, FormatFlushableIngest,
+	case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
-		FormatPrePebblev1MarkedCompacted:
 		return sstable.TableFormatPebblev3
+	case FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
+		return sstable.TableFormatPebblev4
 	default:
 		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
 	}
@@ -186,14 +210,27 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
 		FormatRangeKeys:
 		return sstable.TableFormatLevelDB
 	case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
-		FormatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
+		formatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
-		FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
+		FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
+		FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
 		return sstable.TableFormatPebblev1
 	default:
 		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
 	}
 }
 
+// orderingInvariants returns an enum encoding the set of invariants that must
+// hold within the receiver format major version. Invariants only get stricter
+// as the format major version advances, so it is okay to retrieve the
+// invariants from the current format major version and by the time the
+// invariants are enforced, the format major version has advanced.
+func (v FormatMajorVersion) orderingInvariants() manifest.OrderingInvariants {
+	if v < FormatSplitUserKeysMarkedCompacted {
+		return manifest.AllowSplitUserKeys
+	}
+	return manifest.ProhibitSplitUserKeys
+}
+
 // formatMajorVersionMigrations defines the migrations from one format
 // major version to the next. Each migration is defined as a closure
 // which will be invoked on the database before the new format major
@@ -219,7 +256,7 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 		// guaranteed to exist, because we unconditionally locate it
 		// during Open.
 		manifestFileNum := d.mu.versions.manifestFileNum
-		filename := base.MakeFilename(fileTypeManifest, manifestFileNum)
+		filename := base.MakeFilename(fileTypeManifest, manifestFileNum.DiskFileNum())
 		if err := d.mu.versions.manifestMarker.Move(filename); err != nil {
 			return errors.Wrap(err, "moving manifest marker")
 		}
@@ -257,7 +294,7 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 		// version that does not know about format major versions
 		// attempts to open the database, it will error avoiding
 		// accidental corruption.
-		if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, 0); err != nil {
+		if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, base.FileNum(0).DiskFileNum()); err != nil {
 			return err
 		}
 		return d.finalizeFormatVersUpgrade(FormatVersioned)
@@ -301,9 +338,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 		}
 		return d.finalizeFormatVersUpgrade(FormatPrePebblev1Marked)
 	},
-	FormatUnusedPrePebblev1MarkedCompacted: func(d *DB) error {
+	formatUnusedPrePebblev1MarkedCompacted: func(d *DB) error {
 		// Intentional no-op.
-		return d.finalizeFormatVersUpgrade(FormatUnusedPrePebblev1MarkedCompacted)
+		return d.finalizeFormatVersUpgrade(formatUnusedPrePebblev1MarkedCompacted)
 	},
 	FormatSSTableValueBlocks: func(d *DB) error {
 		return d.finalizeFormatVersUpgrade(FormatSSTableValueBlocks)
@@ -320,6 +357,12 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 		}
 		return d.finalizeFormatVersUpgrade(FormatPrePebblev1MarkedCompacted)
 	},
+	FormatDeleteSizedAndObsolete: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatDeleteSizedAndObsolete)
+	},
+	FormatVirtualSSTables: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatVirtualSSTables)
+	},
 }
 
 const formatVersionMarkerName = `format-version`
@@ -342,7 +385,7 @@ func lookupFormatMajorVersion(
 	if vers == FormatDefault {
 		return 0, nil, errors.Newf("pebble: default format major version should not persisted", vers)
 	}
-	if vers > FormatNewest {
+	if vers > internalFormatNewest {
 		return 0, nil, errors.Newf("pebble: database %q written in format major version %d", dirname, vers)
 	}
 	return vers, m, nil
@@ -376,8 +419,8 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro
 	if d.opts.ReadOnly {
 		return ErrReadOnly
 	}
-	if formatVers > FormatNewest {
+	if formatVers > internalFormatNewest {
-		// Guard against accidentally forgetting to update FormatNewest.
+		// Guard against accidentally forgetting to update internalFormatNewest.
 		return errors.Errorf("pebble: unknown format version %d", formatVers)
 	}
 	if currentVers := d.FormatMajorVersion(); currentVers > formatVers {
@@ -519,17 +562,24 @@ var markFilesPrePebblev1 = func(tc *tableCacheContainer) findFilesFunc {
 		for l := numLevels - 1; l > 0; l-- {
 			iter := v.Levels[l].Iter()
 			for f := iter.First(); f != nil; f = iter.Next() {
-				err = tc.withReader(f, func(r *sstable.Reader) error {
+				if f.Virtual {
-					tf, err := r.TableFormat()
+					// Any physical sstable which has been virtualized must
-					if err != nil {
+					// have already undergone this migration, and we don't
-						return err
+					// need to worry about the virtual sstable themselves.
-					}
+					panic("pebble: unexpected virtual sstable during migration")
-					if tf < sstable.TableFormatPebblev1 {
+				}
-						found = true
+				err = tc.withReader(
-						files[l] = append(files[l], f)
+					f.PhysicalMeta(), func(r *sstable.Reader) error {
-					}
+						tf, err := r.TableFormat()
-					return nil
+						if err != nil {
-				})
+							return err
+						}
+						if tf < sstable.TableFormatPebblev1 {
+							found = true
+							files[l] = append(files[l], f)
+						}
+						return nil
+					})
 				if err != nil {
 					return
 				}

vendor/github.com/cockroachdb/pebble/get_iter.go

@@ -19,8 +19,7 @@ import (
 // lazily.
 type getIter struct {
 	logger       Logger
-	cmp          Compare
+	comparer     *Comparer
-	equal        Equal
 	newIters     tableNewIters
 	snapshot     uint64
 	key          []byte
@@ -83,7 +82,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 			// key. Every call to levelIter.Next() potentially switches to a new
 			// table and thus reinitializes rangeDelIter.
 			if g.rangeDelIter != nil {
-				g.tombstone = keyspan.Get(g.cmp, g.rangeDelIter, g.key)
+				g.tombstone = keyspan.Get(g.comparer.Compare, g.rangeDelIter, g.key)
 				if g.err = g.rangeDelIter.Close(); g.err != nil {
 					return nil, base.LazyValue{}
 				}
@@ -101,7 +100,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 					g.iter = nil
 					return nil, base.LazyValue{}
 				}
-				if g.equal(g.key, key.UserKey) {
+				if g.comparer.Equal(g.key, key.UserKey) {
 					if !key.Visible(g.snapshot, base.InternalKeySeqNumMax) {
 						g.iterKey, g.iterValue = g.iter.Next()
 						continue
@@ -158,12 +157,25 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 			if n := len(g.l0); n > 0 {
 				files := g.l0[n-1].Iter()
 				g.l0 = g.l0[:n-1]
-				iterOpts := IterOptions{logger: g.logger}
+				iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
-				g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters,
+				g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters,
 					files, manifest.L0Sublevel(n), internalIterOpts{})
 				g.levelIter.initRangeDel(&g.rangeDelIter)
+				bc := levelIterBoundaryContext{}
+				g.levelIter.initBoundaryContext(&bc)
 				g.iter = &g.levelIter
-				g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone)
+
+				// Compute the key prefix for bloom filtering if split function is
+				// specified, or use the user key as default.
+				prefix := g.key
+				if g.comparer.Split != nil {
+					prefix = g.key[:g.comparer.Split(g.key)]
+				}
+				g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
+				if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {
+					g.iterKey = nil
+					g.iterValue = base.LazyValue{}
+				}
 				continue
 			}
 			g.level++
@@ -177,13 +189,26 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 			continue
 		}
 
-		iterOpts := IterOptions{logger: g.logger}
+		iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
-		g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters,
+		g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters,
 			g.version.Levels[g.level].Iter(), manifest.Level(g.level), internalIterOpts{})
 		g.levelIter.initRangeDel(&g.rangeDelIter)
+		bc := levelIterBoundaryContext{}
+		g.levelIter.initBoundaryContext(&bc)
 		g.level++
 		g.iter = &g.levelIter
-		g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone)
+
+		// Compute the key prefix for bloom filtering if split function is
+		// specified, or use the user key as default.
+		prefix := g.key
+		if g.comparer.Split != nil {
+			prefix = g.key[:g.comparer.Split(g.key)]
+		}
+		g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
+		if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {
+			g.iterKey = nil
+			g.iterValue = base.LazyValue{}
+		}
 	}
 }
 

vendor/github.com/cockroachdb/pebble/internal.go

@@ -22,7 +22,8 @@ const (
 	InternalKeyKindRangeKeySet     = base.InternalKeyKindRangeKeySet
 	InternalKeyKindRangeKeyUnset   = base.InternalKeyKindRangeKeyUnset
 	InternalKeyKindRangeKeyDelete  = base.InternalKeyKindRangeKeyDelete
-	InternalKeyKindIngestSST       = base.InternalKeyIngestSST
+	InternalKeyKindIngestSST       = base.InternalKeyKindIngestSST
+	InternalKeyKindDeleteSized     = base.InternalKeyKindDeleteSized
 	InternalKeyKindInvalid         = base.InternalKeyKindInvalid
 	InternalKeySeqNumBatch         = base.InternalKeySeqNumBatch
 	InternalKeySeqNumMax           = base.InternalKeySeqNumMax

vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go

@@ -18,22 +18,21 @@
 package arenaskl
 
 import (
-	"math"
 	"sync/atomic"
 	"unsafe"
 
 	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/constants"
+	"github.com/cockroachdb/pebble/internal/invariants"
 )
 
 // Arena is lock-free.
 type Arena struct {
-	n   uint64
+	n   atomic.Uint64
 	buf []byte
 }
 
-const (
+const nodeAlignment = 4
-	align4 = 3
-)
 
 var (
 	// ErrArenaFull indicates that the arena is full and cannot perform any more
@@ -44,20 +43,28 @@ var (
 // NewArena allocates a new arena using the specified buffer as the backing
 // store.
 func NewArena(buf []byte) *Arena {
-	// Don't store data at position 0 in order to reserve offset=0 as a kind
+	if len(buf) > constants.MaxUint32OrInt {
-	// of nil pointer.
+		if invariants.Enabled {
-	return &Arena{
+			panic(errors.AssertionFailedf("attempting to create arena of size %d", len(buf)))
-		n:   1,
+		}
+		buf = buf[:constants.MaxUint32OrInt]
+	}
+	a := &Arena{
 		buf: buf,
 	}
+	// We don't store data at position 0 in order to reserve offset=0 as a kind of
+	// nil pointer.
+	a.n.Store(1)
+	return a
 }
 
 // Size returns the number of bytes allocated by the arena.
 func (a *Arena) Size() uint32 {
-	s := atomic.LoadUint64(&a.n)
+	s := a.n.Load()
-	if s > math.MaxUint32 {
+	if s > constants.MaxUint32OrInt {
-		// Saturate at MaxUint32.
+		// The last failed allocation can push the size higher than len(a.buf).
-		return math.MaxUint32
+		// Saturate at the maximum representable offset.
+		return constants.MaxUint32OrInt
 	}
 	return uint32(s)
 }
@@ -67,24 +74,33 @@ func (a *Arena) Capacity() uint32 {
 	return uint32(len(a.buf))
 }
 
-func (a *Arena) alloc(size, align, overflow uint32) (uint32, uint32, error) {
+// alloc allocates a buffer of the given size and with the given alignment
+// (which must be a power of 2).
+//
+// If overflow is not 0, it also ensures that many bytes after the buffer are
+// inside the arena (this is used for structures that are larger than the
+// requested size but don't use those extra bytes).
+func (a *Arena) alloc(size, alignment, overflow uint32) (uint32, uint32, error) {
+	if invariants.Enabled && (alignment&(alignment-1)) != 0 {
+		panic(errors.AssertionFailedf("invalid alignment %d", alignment))
+	}
 	// Verify that the arena isn't already full.
-	origSize := atomic.LoadUint64(&a.n)
+	origSize := a.n.Load()
 	if int(origSize) > len(a.buf) {
 		return 0, 0, ErrArenaFull
 	}
 
 	// Pad the allocation with enough bytes to ensure the requested alignment.
-	padded := uint32(size) + align
+	padded := uint64(size) + uint64(alignment) - 1
 
-	newSize := atomic.AddUint64(&a.n, uint64(padded))
+	newSize := a.n.Add(padded)
-	if int(newSize)+int(overflow) > len(a.buf) {
+	if newSize+uint64(overflow) > uint64(len(a.buf)) {
 		return 0, 0, ErrArenaFull
 	}
 
 	// Return the aligned offset.
-	offset := (uint32(newSize) - padded + align) & ^align
+	offset := (uint32(newSize) - size) & ^(alignment - 1)
-	return offset, padded, nil
+	return offset, uint32(padded), nil
 }
 
 func (a *Arena) getBytes(offset uint32, size uint32) []byte {

vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go

@@ -18,7 +18,6 @@
 package arenaskl
 
 import (
-	"encoding/binary"
 	"sync"
 
 	"github.com/cockroachdb/pebble/internal/base"
@@ -240,17 +239,8 @@ func (it *Iterator) SetBounds(lower, upper []byte) {
 }
 
 func (it *Iterator) decodeKey() {
-	b := it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
+	it.key.UserKey = it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
-	// This is a manual inline of base.DecodeInternalKey, because the Go compiler
+	it.key.Trailer = it.nd.keyTrailer
-	// seems to refuse to automatically inline it currently.
-	l := len(b) - 8
-	if l >= 0 {
-		it.key.Trailer = binary.LittleEndian.Uint64(b[l:])
-		it.key.UserKey = b[:l:l]
-	} else {
-		it.key.Trailer = uint64(base.InternalKeyKindInvalid)
-		it.key.UserKey = nil
-	}
 }
 
 func (it *Iterator) seekForBaseSplice(key []byte) (prev, next *node, found bool) {

vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go

@@ -29,25 +29,27 @@ import (
 // is used here. If a key/value overflows a uint32, it should not be added to
 // the skiplist.
 func MaxNodeSize(keySize, valueSize uint32) uint64 {
-	return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + align4
+	const maxPadding = nodeAlignment - 1
+	return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + maxPadding
 }
 
 type links struct {
-	nextOffset uint32
+	nextOffset atomic.Uint32
-	prevOffset uint32
+	prevOffset atomic.Uint32
 }
 
 func (l *links) init(prevOffset, nextOffset uint32) {
-	l.nextOffset = nextOffset
+	l.nextOffset.Store(nextOffset)
-	l.prevOffset = prevOffset
+	l.prevOffset.Store(prevOffset)
 }
 
 type node struct {
 	// Immutable fields, so no need to lock to access key.
-	keyOffset uint32
+	keyOffset  uint32
-	keySize   uint32
+	keySize    uint32
-	valueSize uint32
+	keyTrailer uint64
-	allocSize uint32
+	valueSize  uint32
+	allocSize  uint32
 
 	// Most nodes do not need to use the full height of the tower, since the
 	// probability of each successive level decreases exponentially. Because
@@ -65,7 +67,7 @@ func newNode(
 	if height < 1 || height > maxHeight {
 		panic("height cannot be less than one or greater than the max height")
 	}
-	keySize := key.Size()
+	keySize := len(key.UserKey)
 	if int64(keySize) > math.MaxUint32 {
 		panic("key is too large")
 	}
@@ -81,8 +83,8 @@ func newNode(
 	if err != nil {
 		return
 	}
-
+	nd.keyTrailer = key.Trailer
-	key.Encode(nd.getKeyBytes(arena))
+	copy(nd.getKeyBytes(arena), key.UserKey)
 	copy(nd.getValue(arena), value)
 	return
 }
@@ -93,7 +95,7 @@ func newRawNode(arena *Arena, height uint32, keySize, valueSize uint32) (nd *nod
 	unusedSize := uint32((maxHeight - int(height)) * linksSize)
 	nodeSize := uint32(maxNodeSize) - unusedSize
 
-	nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, align4, unusedSize)
+	nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, nodeAlignment, unusedSize)
 	if err != nil {
 		return
 	}
@@ -115,17 +117,17 @@ func (n *node) getValue(arena *Arena) []byte {
 }
 
 func (n *node) nextOffset(h int) uint32 {
-	return atomic.LoadUint32(&n.tower[h].nextOffset)
+	return n.tower[h].nextOffset.Load()
 }
 
 func (n *node) prevOffset(h int) uint32 {
-	return atomic.LoadUint32(&n.tower[h].prevOffset)
+	return n.tower[h].prevOffset.Load()
 }
 
 func (n *node) casNextOffset(h int, old, val uint32) bool {
-	return atomic.CompareAndSwapUint32(&n.tower[h].nextOffset, old, val)
+	return n.tower[h].nextOffset.CompareAndSwap(old, val)
 }
 
 func (n *node) casPrevOffset(h int, old, val uint32) bool {
-	return atomic.CompareAndSwapUint32(&n.tower[h].prevOffset, old, val)
+	return n.tower[h].prevOffset.CompareAndSwap(old, val)
 }

vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go

@@ -44,7 +44,6 @@ Key differences:
 package arenaskl // import "github.com/cockroachdb/pebble/internal/arenaskl"
 
 import (
-	"encoding/binary"
 	"math"
 	"runtime"
 	"sync/atomic"
@@ -68,7 +67,7 @@ const (
 // keys.
 var ErrRecordExists = errors.New("record with this key already exists")
 
-// Skiplist is a fast, cocnurrent skiplist implementation that supports forward
+// Skiplist is a fast, concurrent skiplist implementation that supports forward
 // and backward iteration. See batchskl.Skiplist for a non-concurrent
 // skiplist. Keys and values are immutable once added to the skiplist and
 // deletion is not supported. Instead, higher-level code is expected to add new
@@ -80,7 +79,7 @@ type Skiplist struct {
 	cmp    base.Compare
 	head   *node
 	tail   *node
-	height uint32 // Current height. 1 <= height <= maxHeight. CAS.
+	height atomic.Uint32 // Current height. 1 <= height <= maxHeight. CAS.
 
 	// If set to true by tests, then extra delays are added to make it easier to
 	// detect unusual race conditions.
@@ -140,22 +139,22 @@ func (s *Skiplist) Reset(arena *Arena, cmp base.Compare) {
 	headOffset := arena.getPointerOffset(unsafe.Pointer(head))
 	tailOffset := arena.getPointerOffset(unsafe.Pointer(tail))
 	for i := 0; i < maxHeight; i++ {
-		head.tower[i].nextOffset = tailOffset
+		head.tower[i].nextOffset.Store(tailOffset)
-		tail.tower[i].prevOffset = headOffset
+		tail.tower[i].prevOffset.Store(headOffset)
 	}
 
 	*s = Skiplist{
-		arena:  arena,
+		arena: arena,
-		cmp:    cmp,
+		cmp:   cmp,
-		head:   head,
+		head:  head,
-		tail:   tail,
+		tail:  tail,
-		height: 1,
 	}
+	s.height.Store(1)
 }
 
 // Height returns the height of the highest tower within any of the nodes that
 // have ever been allocated as part of this skiplist.
-func (s *Skiplist) Height() uint32 { return atomic.LoadUint32(&s.height) }
+func (s *Skiplist) Height() uint32 { return s.height.Load() }
 
 // Arena returns the arena backing this skiplist.
 func (s *Skiplist) Arena() *Arena { return s.arena }
@@ -327,7 +326,7 @@ func (s *Skiplist) newNode(
 	// Try to increase s.height via CAS.
 	listHeight := s.Height()
 	for height > listHeight {
-		if atomic.CompareAndSwapUint32(&s.height, listHeight, height) {
+		if s.height.CompareAndSwap(listHeight, height) {
 			// Successfully increased skiplist.height.
 			break
 		}
@@ -412,26 +411,19 @@ func (s *Skiplist) findSpliceForLevel(
 
 		offset, size := next.keyOffset, next.keySize
 		nextKey := s.arena.buf[offset : offset+size]
-		n := int32(size) - 8
+		cmp := s.cmp(key.UserKey, nextKey)
-		cmp := s.cmp(key.UserKey, nextKey[:n])
 		if cmp < 0 {
 			// We are done for this level, since prev.key < key < next.key.
 			break
 		}
 		if cmp == 0 {
 			// User-key equality.
-			var nextTrailer uint64
+			if key.Trailer == next.keyTrailer {
-			if n >= 0 {
-				nextTrailer = binary.LittleEndian.Uint64(nextKey[n:])
-			} else {
-				nextTrailer = uint64(base.InternalKeyKindInvalid)
-			}
-			if key.Trailer == nextTrailer {
 				// Internal key equality.
 				found = true
 				break
 			}
-			if key.Trailer > nextTrailer {
+			if key.Trailer > next.keyTrailer {
 				// We are done for this level, since prev.key < key < next.key.
 				break
 			}
@@ -446,8 +438,7 @@ func (s *Skiplist) findSpliceForLevel(
 
 func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
 	ndKey := s.arena.buf[nd.keyOffset : nd.keyOffset+nd.keySize]
-	n := int32(nd.keySize) - 8
+	cmp := s.cmp(ndKey, key.UserKey)
-	cmp := s.cmp(ndKey[:n], key.UserKey)
 	if cmp < 0 {
 		return true
 	}
@@ -455,25 +446,19 @@ func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
 		return false
 	}
 	// User-key equality.
-	var ndTrailer uint64
+	if key.Trailer == nd.keyTrailer {
-	if n >= 0 {
-		ndTrailer = binary.LittleEndian.Uint64(ndKey[n:])
-	} else {
-		ndTrailer = uint64(base.InternalKeyKindInvalid)
-	}
-	if key.Trailer == ndTrailer {
 		// Internal key equality.
 		return false
 	}
-	return key.Trailer < ndTrailer
+	return key.Trailer < nd.keyTrailer
 }
 
 func (s *Skiplist) getNext(nd *node, h int) *node {
-	offset := atomic.LoadUint32(&nd.tower[h].nextOffset)
+	offset := nd.tower[h].nextOffset.Load()
 	return (*node)(s.arena.getPointer(offset))
 }
 
 func (s *Skiplist) getPrev(nd *node, h int) *node {
-	offset := atomic.LoadUint32(&nd.tower[h].prevOffset)
+	offset := nd.tower[h].prevOffset.Load()
 	return (*node)(s.arena.getPointer(offset))
 }

vendor/github.com/cockroachdb/pebble/internal/base/filenames.go

@@ -20,6 +20,41 @@ type FileNum uint64
 // String returns a string representation of the file number.
 func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
 
+// SafeFormat implements redact.SafeFormatter.
+func (fn FileNum) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Printf("%06d", redact.SafeUint(fn))
+}
+
+// DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
+// called if the caller can ensure that the FileNum belongs to a physical file
+// on disk. These could be manifests, log files, physical sstables on disk, the
+// options file, but not virtual sstables.
+func (fn FileNum) DiskFileNum() DiskFileNum {
+	return DiskFileNum{fn}
+}
+
+// A DiskFileNum is just a FileNum belonging to a file which exists on disk.
+// Note that a FileNum is an internal DB identifier and it could belong to files
+// which don't exist on disk. An example would be virtual sstable FileNums.
+// Converting a DiskFileNum to a FileNum is always valid, whereas converting a
+// FileNum to DiskFileNum may not be valid and care should be taken to prove
+// that the FileNum actually exists on disk.
+type DiskFileNum struct {
+	fn FileNum
+}
+
+func (dfn DiskFileNum) String() string { return dfn.fn.String() }
+
+// SafeFormat implements redact.SafeFormatter.
+func (dfn DiskFileNum) SafeFormat(w redact.SafePrinter, verb rune) {
+	dfn.fn.SafeFormat(w, verb)
+}
+
+// FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
+func (dfn DiskFileNum) FileNum() FileNum {
+	return dfn.fn
+}
+
 // FileType enumerates the types of files found in a DB.
 type FileType int
 
@@ -36,92 +71,92 @@ const (
 )
 
 // MakeFilename builds a filename from components.
-func MakeFilename(fileType FileType, fileNum FileNum) string {
+func MakeFilename(fileType FileType, dfn DiskFileNum) string {
 	switch fileType {
 	case FileTypeLog:
-		return fmt.Sprintf("%s.log", fileNum)
+		return fmt.Sprintf("%s.log", dfn)
 	case FileTypeLock:
 		return "LOCK"
 	case FileTypeTable:
-		return fmt.Sprintf("%s.sst", fileNum)
+		return fmt.Sprintf("%s.sst", dfn)
 	case FileTypeManifest:
-		return fmt.Sprintf("MANIFEST-%s", fileNum)
+		return fmt.Sprintf("MANIFEST-%s", dfn)
 	case FileTypeCurrent:
 		return "CURRENT"
 	case FileTypeOptions:
-		return fmt.Sprintf("OPTIONS-%s", fileNum)
+		return fmt.Sprintf("OPTIONS-%s", dfn)
 	case FileTypeOldTemp:
-		return fmt.Sprintf("CURRENT.%s.dbtmp", fileNum)
+		return fmt.Sprintf("CURRENT.%s.dbtmp", dfn)
 	case FileTypeTemp:
-		return fmt.Sprintf("temporary.%s.dbtmp", fileNum)
+		return fmt.Sprintf("temporary.%s.dbtmp", dfn)
 	}
 	panic("unreachable")
 }
 
 // MakeFilepath builds a filepath from components.
-func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, fileNum FileNum) string {
+func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, dfn DiskFileNum) string {
-	return fs.PathJoin(dirname, MakeFilename(fileType, fileNum))
+	return fs.PathJoin(dirname, MakeFilename(fileType, dfn))
 }
 
 // ParseFilename parses the components from a filename.
-func ParseFilename(fs vfs.FS, filename string) (fileType FileType, fileNum FileNum, ok bool) {
+func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileNum, ok bool) {
 	filename = fs.PathBase(filename)
 	switch {
 	case filename == "CURRENT":
-		return FileTypeCurrent, 0, true
+		return FileTypeCurrent, DiskFileNum{0}, true
 	case filename == "LOCK":
-		return FileTypeLock, 0, true
+		return FileTypeLock, DiskFileNum{0}, true
 	case strings.HasPrefix(filename, "MANIFEST-"):
-		fileNum, ok = parseFileNum(filename[len("MANIFEST-"):])
+		dfn, ok = parseDiskFileNum(filename[len("MANIFEST-"):])
 		if !ok {
 			break
 		}
-		return FileTypeManifest, fileNum, true
+		return FileTypeManifest, dfn, true
 	case strings.HasPrefix(filename, "OPTIONS-"):
-		fileNum, ok = parseFileNum(filename[len("OPTIONS-"):])
+		dfn, ok = parseDiskFileNum(filename[len("OPTIONS-"):])
 		if !ok {
 			break
 		}
-		return FileTypeOptions, fileNum, ok
+		return FileTypeOptions, dfn, ok
 	case strings.HasPrefix(filename, "CURRENT.") && strings.HasSuffix(filename, ".dbtmp"):
 		s := strings.TrimSuffix(filename[len("CURRENT."):], ".dbtmp")
-		fileNum, ok = parseFileNum(s)
+		dfn, ok = parseDiskFileNum(s)
 		if !ok {
 			break
 		}
-		return FileTypeOldTemp, fileNum, ok
+		return FileTypeOldTemp, dfn, ok
 	case strings.HasPrefix(filename, "temporary.") && strings.HasSuffix(filename, ".dbtmp"):
 		s := strings.TrimSuffix(filename[len("temporary."):], ".dbtmp")
-		fileNum, ok = parseFileNum(s)
+		dfn, ok = parseDiskFileNum(s)
 		if !ok {
 			break
 		}
-		return FileTypeTemp, fileNum, ok
+		return FileTypeTemp, dfn, ok
 	default:
 		i := strings.IndexByte(filename, '.')
 		if i < 0 {
 			break
 		}
-		fileNum, ok = parseFileNum(filename[:i])
+		dfn, ok = parseDiskFileNum(filename[:i])
 		if !ok {
 			break
 		}
 		switch filename[i+1:] {
 		case "sst":
-			return FileTypeTable, fileNum, true
+			return FileTypeTable, dfn, true
 		case "log":
-			return FileTypeLog, fileNum, true
+			return FileTypeLog, dfn, true
 		}
 	}
-	return 0, fileNum, false
+	return 0, dfn, false
 }
 
-func parseFileNum(s string) (fileNum FileNum, ok bool) {
+func parseDiskFileNum(s string) (dfn DiskFileNum, ok bool) {
 	u, err := strconv.ParseUint(s, 10, 64)
 	if err != nil {
-		return fileNum, false
+		return dfn, false
 	}
-	return FileNum(u), true
+	return DiskFileNum{FileNum(u)}, true
 }
 
 // A Fataler fatals a process with a message when called.

vendor/github.com/cockroachdb/pebble/internal/base/internal.go

@@ -9,6 +9,17 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+
+	"github.com/cockroachdb/redact"
+)
+
+const (
+	// SeqNumZero is the zero sequence number, set by compactions if they can
+	// guarantee there are no keys underneath an internal key.
+	SeqNumZero = uint64(0)
+	// SeqNumStart is the first sequence number assigned to a key. Sequence
+	// numbers 1-9 are reserved for potential future use.
+	SeqNumStart = uint64(10)
 )
 
 // InternalKeyKind enumerates the kind of key: a deletion tombstone, a set
@@ -24,6 +35,14 @@ const (
 	//InternalKeyKindColumnFamilyDeletion     InternalKeyKind = 4
 	//InternalKeyKindColumnFamilyValue        InternalKeyKind = 5
 	//InternalKeyKindColumnFamilyMerge        InternalKeyKind = 6
+
+	// InternalKeyKindSingleDelete (SINGLEDEL) is a performance optimization
+	// solely for compactions (to reduce write amp and space amp). Readers other
+	// than compactions should treat SINGLEDEL as equivalent to a DEL.
+	// Historically, it was simpler for readers other than compactions to treat
+	// SINGLEDEL as equivalent to DEL, but as of the introduction of
+	// InternalKeyKindSSTableInternalObsoleteBit, this is also necessary for
+	// correctness.
 	InternalKeyKindSingleDelete InternalKeyKind = 7
 	//InternalKeyKindColumnFamilySingleDelete InternalKeyKind = 8
 	//InternalKeyKindBeginPrepareXID          InternalKeyKind = 9
@@ -60,14 +79,21 @@ const (
 	InternalKeyKindRangeKeyUnset InternalKeyKind = 20
 	InternalKeyKindRangeKeySet   InternalKeyKind = 21
 
-	// InternalKeyIngestSST is used to distinguish a batch that corresponds to
+	// InternalKeyKindIngestSST is used to distinguish a batch that corresponds to
 	// the WAL entry for ingested sstables that are added to the flushable
 	// queue. This InternalKeyKind cannot appear, amongst other key kinds in a
 	// batch, or in an sstable.
-	InternalKeyIngestSST InternalKeyKind = 22
+	InternalKeyKindIngestSST InternalKeyKind = 22
 
-	// This maximum value isn't part of the file format. It's unlikely,
+	// InternalKeyKindDeleteSized keys behave identically to
-	// but future extensions may increase this value.
+	// InternalKeyKindDelete keys, except that they hold an associated uint64
+	// value indicating the (len(key)+len(value)) of the shadowed entry the
+	// tombstone is expected to delete. This value is used to inform compaction
+	// heuristics, but is not required to be accurate for correctness.
+	InternalKeyKindDeleteSized InternalKeyKind = 23
+
+	// This maximum value isn't part of the file format. Future extensions may
+	// increase this value.
 	//
 	// When constructing an internal key to pass to DB.Seek{GE,LE},
 	// internalKeyComparer sorts decreasing by kind (after sorting increasing by
@@ -75,14 +101,19 @@ const (
 	// which sorts 'less than or equal to' any other valid internalKeyKind, when
 	// searching for any kind of internal key formed by a certain user key and
 	// seqNum.
-	InternalKeyKindMax InternalKeyKind = 22
+	InternalKeyKindMax InternalKeyKind = 23
+
+	// Internal to the sstable format. Not exposed by any sstable iterator.
+	// Declared here to prevent definition of valid key kinds that set this bit.
+	InternalKeyKindSSTableInternalObsoleteBit  InternalKeyKind = 64
+	InternalKeyKindSSTableInternalObsoleteMask InternalKeyKind = 191
 
 	// InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a
 	// zero sequence number.
-	InternalKeyZeroSeqnumMaxTrailer = uint64(InternalKeyKindInvalid)
+	InternalKeyZeroSeqnumMaxTrailer = uint64(255)
 
 	// A marker for an invalid key.
-	InternalKeyKindInvalid InternalKeyKind = 255
+	InternalKeyKindInvalid InternalKeyKind = InternalKeyKindSSTableInternalObsoleteMask
 
 	// InternalKeySeqNumBatch is a bit that is set on batch sequence numbers
 	// which prevents those entries from being excluded from iteration.
@@ -105,6 +136,9 @@ const (
 	InternalKeyBoundaryRangeKey = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeKeySet)
 )
 
+// Assert InternalKeyKindSSTableInternalObsoleteBit > InternalKeyKindMax
+const _ = uint(InternalKeyKindSSTableInternalObsoleteBit - InternalKeyKindMax - 1)
+
 var internalKeyKindNames = []string{
 	InternalKeyKindDelete:         "DEL",
 	InternalKeyKindSet:            "SET",
@@ -117,7 +151,8 @@ var internalKeyKindNames = []string{
 	InternalKeyKindRangeKeySet:    "RANGEKEYSET",
 	InternalKeyKindRangeKeyUnset:  "RANGEKEYUNSET",
 	InternalKeyKindRangeKeyDelete: "RANGEKEYDEL",
-	InternalKeyIngestSST:          "INGESTSST",
+	InternalKeyKindIngestSST:      "INGESTSST",
+	InternalKeyKindDeleteSized:    "DELSIZED",
 	InternalKeyKindInvalid:        "INVALID",
 }
 
@@ -128,6 +163,11 @@ func (k InternalKeyKind) String() string {
 	return fmt.Sprintf("UNKNOWN:%d", k)
 }
 
+// SafeFormat implements redact.SafeFormatter.
+func (k InternalKeyKind) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Print(redact.SafeString(k.String()))
+}
+
 // InternalKey is a key used for the in-memory and on-disk partial DBs that
 // make up a pebble DB.
 //
@@ -194,6 +234,7 @@ var kindsMap = map[string]InternalKeyKind{
 	"DEL":           InternalKeyKindDelete,
 	"SINGLEDEL":     InternalKeyKindSingleDelete,
 	"RANGEDEL":      InternalKeyKindRangeDelete,
+	"LOGDATA":       InternalKeyKindLogData,
 	"SET":           InternalKeyKindSet,
 	"MERGE":         InternalKeyKindMerge,
 	"INVALID":       InternalKeyKindInvalid,
@@ -202,6 +243,8 @@ var kindsMap = map[string]InternalKeyKind{
 	"RANGEKEYSET":   InternalKeyKindRangeKeySet,
 	"RANGEKEYUNSET": InternalKeyKindRangeKeyUnset,
 	"RANGEKEYDEL":   InternalKeyKindRangeKeyDelete,
+	"INGESTSST":     InternalKeyKindIngestSST,
+	"DELSIZED":      InternalKeyKindDeleteSized,
 }
 
 // ParseInternalKey parses the string representation of an internal key. The
@@ -338,6 +381,11 @@ func (k InternalKey) SeqNum() uint64 {
 	return k.Trailer >> 8
 }
 
+// SeqNumFromTrailer returns the sequence number component of a trailer.
+func SeqNumFromTrailer(t uint64) uint64 {
+	return t >> 8
+}
+
 // Visible returns true if the key is visible at the specified snapshot
 // sequence number.
 func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool {
@@ -395,6 +443,13 @@ func (k InternalKey) Clone() InternalKey {
 	}
 }
 
+// CopyFrom converts this InternalKey into a clone of the passed-in InternalKey,
+// reusing any space already used for the current UserKey.
+func (k *InternalKey) CopyFrom(k2 InternalKey) {
+	k.UserKey = append(k.UserKey[:0], k2.UserKey...)
+	k.Trailer = k2.Trailer
+}
+
 // String returns a string representation of the key.
 func (k InternalKey) String() string {
 	return fmt.Sprintf("%s#%d,%d", FormatBytes(k.UserKey), k.SeqNum(), k.Kind())

vendor/github.com/cockroachdb/pebble/internal/base/metrics.go

@@ -53,6 +53,15 @@ func (tm *ThroughputMetric) Rate() int64 {
 		float64(time.Second))
 }
 
+// Utilization returns a percent [0, 1.0] indicating the percent of time
+// work was performed.
+func (tm *ThroughputMetric) Utilization() float64 {
+	if tm.WorkDuration == 0 {
+		return 0
+	}
+	return float64(tm.WorkDuration) / float64(tm.WorkDuration+tm.IdleDuration)
+}
+
 // GaugeSampleMetric is used to measure a gauge value (e.g. queue length) by
 // accumulating samples of that gauge.
 type GaugeSampleMetric struct {

vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go

@@ -63,14 +63,15 @@ import (
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/constants"
 	"golang.org/x/exp/rand"
 )
 
 const (
 	maxHeight    = 20
-	maxNodeSize  = int(unsafe.Sizeof(node{}))
+	maxNodeSize  = uint64(unsafe.Sizeof(node{}))
-	linksSize    = int(unsafe.Sizeof(links{}))
+	linksSize    = uint64(unsafe.Sizeof(links{}))
-	maxNodesSize = math.MaxUint32
+	maxNodesSize = constants.MaxUint32OrInt
 )
 
 var (
@@ -281,7 +282,7 @@ func (s *Skiplist) newNode(
 		panic("height cannot be less than one or greater than the max height")
 	}
 
-	unusedSize := (maxHeight - int(height)) * linksSize
+	unusedSize := uint64(maxHeight-int(height)) * linksSize
 	nodeOffset, err := s.alloc(uint32(maxNodeSize - unusedSize))
 	if err != nil {
 		return 0, err
@@ -296,13 +297,13 @@ func (s *Skiplist) newNode(
 }
 
 func (s *Skiplist) alloc(size uint32) (uint32, error) {
-	offset := len(s.nodes)
+	offset := uint64(len(s.nodes))
 
 	// We only have a need for memory up to offset + size, but we never want
 	// to allocate a node whose tail points into unallocated memory.
 	minAllocSize := offset + maxNodeSize
-	if cap(s.nodes) < minAllocSize {
+	if uint64(cap(s.nodes)) < minAllocSize {
-		allocSize := cap(s.nodes) * 2
+		allocSize := uint64(cap(s.nodes)) * 2
 		if allocSize < minAllocSize {
 			allocSize = minAllocSize
 		}

vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go

@@ -33,7 +33,7 @@ import (
 type fileKey struct {
 	// id is the namespace for fileNums.
 	id      uint64
-	fileNum base.FileNum
+	fileNum base.DiskFileNum
 }
 
 type key struct {
@@ -71,14 +71,12 @@ func (h Handle) Get() []byte {
 
 // Release releases the reference to the cache entry.
 func (h Handle) Release() {
-	if h.value != nil {
+	h.value.release()
-		h.value.release()
-	}
 }
 
 type shard struct {
-	hits   int64
+	hits   atomic.Int64
-	misses int64
+	misses atomic.Int64
 
 	mu sync.RWMutex
 
@@ -113,25 +111,25 @@ type shard struct {
 	countTest int64
 }
 
-func (c *shard) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
+func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
 	c.mu.RLock()
 	var value *Value
 	if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil {
 		value = e.acquireValue()
 		if value != nil {
-			atomic.StoreInt32(&e.referenced, 1)
+			e.referenced.Store(true)
 		}
 	}
 	c.mu.RUnlock()
 	if value == nil {
-		atomic.AddInt64(&c.misses, 1)
+		c.misses.Add(1)
 		return Handle{}
 	}
-	atomic.AddInt64(&c.hits, 1)
+	c.hits.Add(1)
 	return Handle{value: value}
 }
 
-func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle {
+func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
 	if n := value.refs(); n != 1 {
 		panic(fmt.Sprintf("pebble: Value has already been added to the cache: refs=%d", n))
 	}
@@ -160,7 +158,7 @@ func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value
 	case e.peekValue() != nil:
 		// cache entry was a hot or cold page
 		e.setValue(value)
-		atomic.StoreInt32(&e.referenced, 1)
+		e.referenced.Store(true)
 		delta := int64(len(value.buf)) - e.size
 		e.size = int64(len(value.buf))
 		if e.ptype == etHot {
@@ -176,7 +174,7 @@ func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value
 		// cache entry was a test page
 		c.sizeTest -= e.size
 		c.countTest--
-		c.metaDel(e)
+		c.metaDel(e).release()
 		c.metaCheck(e)
 
 		e.size = int64(len(value.buf))
@@ -185,7 +183,7 @@ func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value
 			c.coldTarget = c.targetSize()
 		}
 
-		atomic.StoreInt32(&e.referenced, 0)
+		e.referenced.Store(false)
 		e.setValue(value)
 		e.ptype = etHot
 		if c.metaAdd(k, e) {
@@ -222,7 +220,7 @@ func (c *shard) checkConsistency() {
 }
 
 // Delete deletes the cached value for the specified file and offset.
-func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) {
+func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
 	// The common case is there is nothing to delete, so do a quick check with
 	// shared lock.
 	k := key{fileKey{id, fileNum}, offset}
@@ -233,37 +231,75 @@ func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) {
 		return
 	}
 
-	c.mu.Lock()
+	var deletedValue *Value
-	defer c.mu.Unlock()
+	func() {
+		c.mu.Lock()
+		defer c.mu.Unlock()
 
-	e := c.blocks.Get(k)
+		e := c.blocks.Get(k)
-	if e == nil {
+		if e == nil {
-		return
+			return
-	}
+		}
-	c.metaEvict(e)
+		deletedValue = c.metaEvict(e)
-
+		c.checkConsistency()
-	c.checkConsistency()
+	}()
+	// Now that the mutex has been dropped, release the reference which will
+	// potentially free the memory associated with the previous cached value.
+	deletedValue.release()
 }
 
 // EvictFile evicts all of the cache values for the specified file.
-func (c *shard) EvictFile(id uint64, fileNum base.FileNum) {
+func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
 	fkey := key{fileKey{id, fileNum}, 0}
+	for c.evictFileRun(fkey) {
+		// Sched switch to give another goroutine an opportunity to acquire the
+		// shard mutex.
+		runtime.Gosched()
+	}
+}
+
+func (c *shard) evictFileRun(fkey key) (moreRemaining bool) {
+	// If most of the file's blocks are held in the block cache, evicting all
+	// the blocks may take a while. We don't want to block the entire cache
+	// shard, forcing concurrent readers to wait until we're finished. We drop
+	// the mutex every [blocksPerMutexAcquisition] blocks to give other
+	// goroutines an opportunity to make progress.
+	const blocksPerMutexAcquisition = 5
+	c.mu.Lock()
+
+	// Releasing a value may result in free-ing it back to the memory allocator.
+	// This can have a nontrivial cost that we'd prefer to not pay while holding
+	// the shard mutex, so we collect the evicted values in a local slice and
+	// only release them in a defer after dropping the cache mutex.
+	var obsoleteValuesAlloc [blocksPerMutexAcquisition]*Value
+	obsoleteValues := obsoleteValuesAlloc[:0]
+	defer func() {
+		c.mu.Unlock()
+		for _, v := range obsoleteValues {
+			v.release()
+		}
+	}()
+
 	blocks := c.files.Get(fkey)
 	if blocks == nil {
-		return
+		// No blocks for this file.
-	}
+		return false
-	for b, n := blocks, (*entry)(nil); ; b = n {
-		n = b.fileLink.next
-		c.metaEvict(b)
-		if b == n {
-			break
-		}
 	}
 
-	c.checkConsistency()
+	// b is the current head of the doubly linked list, and n is the entry after b.
+	for b, n := blocks, (*entry)(nil); len(obsoleteValues) < cap(obsoleteValues); b = n {
+		n = b.fileLink.next
+		obsoleteValues = append(obsoleteValues, c.metaEvict(b))
+		if b == n {
+			// b == n represents the case where b was the last entry remaining
+			// in the doubly linked list, which is why it pointed at itself. So
+			// no more entries left.
+			c.checkConsistency()
+			return false
+		}
+	}
+	// Exhausted blocksPerMutexAcquisition.
+	return true
 }
 
 func (c *shard) Free() {
@@ -274,7 +310,7 @@ func (c *shard) Free() {
 	// metaCheck call when the "invariants" build tag is specified.
 	for c.handHot != nil {
 		e := c.handHot
-		c.metaDel(c.handHot)
+		c.metaDel(c.handHot).release()
 		e.free()
 	}
 
@@ -359,12 +395,14 @@ func (c *shard) metaAdd(key key, e *entry) bool {
 
 // Remove the entry from the cache. This removes the entry from the blocks map,
 // the files map, and ensures that hand{Hot,Cold,Test} are not pointing at the
-// entry.
+// entry. Returns the deleted value that must be released, if any.
-func (c *shard) metaDel(e *entry) {
+func (c *shard) metaDel(e *entry) (deletedValue *Value) {
 	if value := e.peekValue(); value != nil {
 		value.ref.trace("metaDel")
 	}
-	e.setValue(nil)
+	// Remove the pointer to the value.
+	deletedValue = e.val
+	e.val = nil
 
 	c.blocks.Delete(e.key)
 	if entriesGoAllocated {
@@ -396,6 +434,7 @@ func (c *shard) metaDel(e *entry) {
 	} else {
 		c.files.Put(fkey, next)
 	}
+	return deletedValue
 }
 
 // Check that the specified entry is not referenced by the cache.
@@ -455,7 +494,7 @@ func (c *shard) metaCheck(e *entry) {
 	}
 }
 
-func (c *shard) metaEvict(e *entry) {
+func (c *shard) metaEvict(e *entry) (evictedValue *Value) {
 	switch e.ptype {
 	case etHot:
 		c.sizeHot -= e.size
@@ -467,9 +506,10 @@ func (c *shard) metaEvict(e *entry) {
 		c.sizeTest -= e.size
 		c.countTest--
 	}
-	c.metaDel(e)
+	evictedValue = c.metaDel(e)
 	c.metaCheck(e)
 	e.free()
+	return evictedValue
 }
 
 func (c *shard) evict() {
@@ -491,8 +531,8 @@ func (c *shard) runHandCold(countColdDebug, sizeColdDebug int64) {
 
 	e := c.handCold
 	if e.ptype == etCold {
-		if atomic.LoadInt32(&e.referenced) == 1 {
+		if e.referenced.Load() {
-			atomic.StoreInt32(&e.referenced, 0)
+			e.referenced.Store(false)
 			e.ptype = etHot
 			c.sizeCold -= e.size
 			c.countCold--
@@ -528,8 +568,8 @@ func (c *shard) runHandHot() {
 
 	e := c.handHot
 	if e.ptype == etHot {
-		if atomic.LoadInt32(&e.referenced) == 1 {
+		if e.referenced.Load() {
-			atomic.StoreInt32(&e.referenced, 0)
+			e.referenced.Store(false)
 		} else {
 			e.ptype = etCold
 			c.sizeHot -= e.size
@@ -564,7 +604,7 @@ func (c *shard) runHandTest() {
 		if c.coldTarget < 0 {
 			c.coldTarget = 0
 		}
-		c.metaDel(e)
+		c.metaDel(e).release()
 		c.metaCheck(e)
 		e.free()
 	}
@@ -587,7 +627,7 @@ type Metrics struct {
 // Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is
 // used for page replacement
 // (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In
-// order to provide better concurrency, 2 x NumCPUs shards are created, with
+// order to provide better concurrency, 4 x NumCPUs shards are created, with
 // each shard being given 1/n of the target cache size. The Clock-PRO algorithm
 // is run independently on each shard.
 //
@@ -621,9 +661,9 @@ type Metrics struct {
 // used in combination by specifying `-tags invariants,tracing`. Note that
 // "tracing" produces a significant slowdown, while "invariants" does not.
 type Cache struct {
-	refs    int64
+	refs    atomic.Int64
 	maxSize int64
-	idAlloc uint64
+	idAlloc atomic.Uint64
 	shards  []shard
 
 	// Traces recorded by Cache.trace. Used for debugging.
@@ -643,17 +683,44 @@ type Cache struct {
 //	defer c.Unref()
 //	d, err := pebble.Open(pebble.Options{Cache: c})
 func New(size int64) *Cache {
-	return newShards(size, 2*runtime.GOMAXPROCS(0))
+	// How many cache shards should we create?
+	//
+	// Note that the probability two processors will try to access the same
+	// shard at the same time increases superlinearly with the number of
+	// processors (Eg, consider the brithday problem where each CPU is a person,
+	// and each shard is a possible birthday).
+	//
+	// We could consider growing the number of shards superlinearly, but
+	// increasing the shard count may reduce the effectiveness of the caching
+	// algorithm if frequently-accessed blocks are insufficiently distributed
+	// across shards. If a shard's size is smaller than a single frequently
+	// scanned sstable, then the shard will be unable to hold the entire
+	// frequently-scanned table in memory despite other shards still holding
+	// infrequently accessed blocks.
+	//
+	// Experimentally, we've observed contention contributing to tail latencies
+	// at 2 shards per processor. For now we use 4 shards per processor,
+	// recognizing this may not be final word.
+	m := 4 * runtime.GOMAXPROCS(0)
+
+	// In tests we can use large CPU machines with small cache sizes and have
+	// many caches in existence at a time. If sharding into m shards would
+	// produce too small shards, constrain the number of shards to 4.
+	const minimumShardSize = 4 << 20 // 4 MiB
+	if m > 4 && int(size)/m < minimumShardSize {
+		m = 4
+	}
+	return newShards(size, m)
 }
 
 func newShards(size int64, shards int) *Cache {
 	c := &Cache{
-		refs:    1,
 		maxSize: size,
-		idAlloc: 1,
 		shards:  make([]shard, shards),
 	}
-	c.trace("alloc", c.refs)
+	c.refs.Store(1)
+	c.idAlloc.Store(1)
+	c.trace("alloc", c.refs.Load())
 	for i := range c.shards {
 		c.shards[i] = shard{
 			maxSize:    size / int64(len(c.shards)),
@@ -669,7 +736,7 @@ func newShards(size int64, shards int) *Cache {
 	// Note: this is a no-op if invariants are disabled or race is enabled.
 	invariants.SetFinalizer(c, func(obj interface{}) {
 		c := obj.(*Cache)
-		if v := atomic.LoadInt64(&c.refs); v != 0 {
+		if v := c.refs.Load(); v != 0 {
 			c.tr.Lock()
 			fmt.Fprintf(os.Stderr,
 				"pebble: cache (%p) has non-zero reference count: %d\n", c, v)
@@ -683,7 +750,7 @@ func newShards(size int64, shards int) *Cache {
 	return c
 }
 
-func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard {
+func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *shard {
 	if id == 0 {
 		panic("pebble: 0 cache ID is invalid")
 	}
@@ -698,10 +765,11 @@ func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard
 		h ^= uint64(id & 0xff)
 		id >>= 8
 	}
+	fileNumVal := uint64(fileNum.FileNum())
 	for i := 0; i < 8; i++ {
 		h *= prime64
-		h ^= uint64(fileNum & 0xff)
+		h ^= uint64(fileNumVal) & 0xff
-		fileNum >>= 8
+		fileNumVal >>= 8
 	}
 	for i := 0; i < 8; i++ {
 		h *= prime64
@@ -715,7 +783,7 @@ func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard
 // Ref adds a reference to the cache. The cache only remains valid as long a
 // reference is maintained to it.
 func (c *Cache) Ref() {
-	v := atomic.AddInt64(&c.refs, 1)
+	v := c.refs.Add(1)
 	if v <= 1 {
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
 	}
@@ -724,7 +792,7 @@ func (c *Cache) Ref() {
 
 // Unref releases a reference on the cache.
 func (c *Cache) Unref() {
-	v := atomic.AddInt64(&c.refs, -1)
+	v := c.refs.Add(-1)
 	c.trace("unref", v)
 	switch {
 	case v < 0:
@@ -738,7 +806,7 @@ func (c *Cache) Unref() {
 
 // Get retrieves the cache value for the specified file and offset, returning
 // nil if no value is present.
-func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
+func (c *Cache) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
 	return c.getShard(id, fileNum, offset).Get(id, fileNum, offset)
 }
 
@@ -746,17 +814,17 @@ func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle {
 // existing value if present. A Handle is returned which provides faster
 // retrieval of the cached value than Get (lock-free and avoidance of the map
 // lookup). The value must have been allocated by Cache.Alloc.
-func (c *Cache) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle {
+func (c *Cache) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle {
 	return c.getShard(id, fileNum, offset).Set(id, fileNum, offset, value)
 }
 
 // Delete deletes the cached value for the specified file and offset.
-func (c *Cache) Delete(id uint64, fileNum base.FileNum, offset uint64) {
+func (c *Cache) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
 	c.getShard(id, fileNum, offset).Delete(id, fileNum, offset)
 }
 
 // EvictFile evicts all of the cache values for the specified file.
-func (c *Cache) EvictFile(id uint64, fileNum base.FileNum) {
+func (c *Cache) EvictFile(id uint64, fileNum base.DiskFileNum) {
 	if id == 0 {
 		panic("pebble: 0 cache ID is invalid")
 	}
@@ -784,14 +852,14 @@ func (c *Cache) Size() int64 {
 // manually managed. The caller MUST either add the value to the cache (via
 // Cache.Set), or release the value (via Cache.Free). Failure to do so will
 // result in a memory leak.
-func (c *Cache) Alloc(n int) *Value {
+func Alloc(n int) *Value {
 	return newValue(n)
 }
 
 // Free frees the specified value. The buffer associated with the value will
 // possibly be reused, making it invalid to use the buffer after calling
 // Free. Do not call Free on a value that has been added to the cache.
-func (c *Cache) Free(v *Value) {
+func Free(v *Value) {
 	if n := v.refs(); n > 1 {
 		panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n))
 	}
@@ -828,8 +896,8 @@ func (c *Cache) Metrics() Metrics {
 		m.Count += int64(s.blocks.Count())
 		m.Size += s.sizeHot + s.sizeCold
 		s.mu.RUnlock()
-		m.Hits += atomic.LoadInt64(&s.hits)
+		m.Hits += s.hits.Load()
-		m.Misses += atomic.LoadInt64(&s.misses)
+		m.Misses += s.misses.Load()
 	}
 	return m
 }
@@ -837,5 +905,5 @@ func (c *Cache) Metrics() Metrics {
 // NewID returns a new ID to be used as a namespace for cached file
 // blocks.
 func (c *Cache) NewID() uint64 {
-	return atomic.AddUint64(&c.idAlloc, 1)
+	return c.idAlloc.Add(1)
 }

vendor/github.com/cockroachdb/pebble/internal/cache/entry.go

@@ -4,6 +4,8 @@
 
 package cache
 
+import "sync/atomic"
+
 type entryType int8
 
 const (
@@ -56,7 +58,7 @@ type entry struct {
 	ptype entryType
 	// referenced is atomically set to indicate that this entry has been accessed
 	// since the last time one of the clock hands swept it.
-	referenced int32
+	referenced atomic.Bool
 	shard      *shard
 	// Reference count for the entry. The entry is freed when the reference count
 	// drops to zero.
@@ -137,9 +139,7 @@ func (e *entry) setValue(v *Value) {
 	}
 	old := e.val
 	e.val = v
-	if old != nil {
+	old.release()
-		old.release()
-	}
 }
 
 func (e *entry) peekValue() *Value {

vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go

@@ -24,7 +24,7 @@ func robinHoodHash(k key, shift uint32) uint32 {
 	const m = 11400714819323198485
 	h := hashSeed
 	h ^= k.id * m
-	h ^= uint64(k.fileNum) * m
+	h ^= uint64(k.fileNum.FileNum()) * m
 	h ^= k.offset * m
 	return uint32(h >> shift)
 }

vendor/github.com/cockroachdb/pebble/internal/cache/value.go

@@ -40,7 +40,7 @@ func (v *Value) acquire() {
 }
 
 func (v *Value) release() {
-	if v.ref.release() {
+	if v != nil && v.ref.release() {
 		v.free()
 	}
 }

vendor/github.com/cockroachdb/pebble/internal/constants/constants.go

@@ -0,0 +1,17 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package constants
+
+const (
+	// oneIf64Bit is 1 on 64-bit platforms and 0 on 32-bit platforms.
+	oneIf64Bit = ^uint(0) >> 63
+
+	// MaxUint32OrInt returns min(MaxUint32, MaxInt), i.e
+	// - MaxUint32 on 64-bit platforms;
+	// - MaxInt on 32-bit platforms.
+	// It is used when slices are limited to Uint32 on 64-bit platforms (the
+	// length limit for slices is naturally MaxInt on 32-bit platforms).
+	MaxUint32OrInt = (1<<31)<<oneIf64Bit - 1
+)

vendor/github.com/cockroachdb/pebble/internal/humanize/humanize.go

@@ -35,11 +35,11 @@ type config struct {
 	suffix []string
 }
 
-// IEC produces human readable representations of integer values in IEC units.
+// Bytes produces human readable representations of byte values in IEC units.
-var IEC = config{1024, []string{" B", " K", " M", " G", " T", " P", " E"}}
+var Bytes = config{1024, []string{"B", "KB", "MB", "GB", "TB", "PB", "EB"}}
 
-// SI produces human readable representations of integer values in SI units.
+// Count produces human readable representations of unitless values in SI units.
-var SI = config{1000, []string{"", " K", " M", " G", " T", " P", " E"}}
+var Count = config{1000, []string{"", "K", "M", "G", "T", "P", "E"}}
 
 // Int64 produces a human readable representation of the value.
 func (c *config) Int64(s int64) FormattedString {
@@ -54,18 +54,6 @@ func (c *config) Uint64(s uint64) FormattedString {
 	return FormattedString(humanate(s, c.base, c.suffix))
 }
 
-// Int64 produces a human readable representation of the value in IEC units
-// (base 1024).
-func Int64(s int64) FormattedString {
-	return IEC.Int64(s)
-}
-
-// Uint64 produces a human readable representation of the value in IEC units
-// (base 1024).
-func Uint64(s uint64) FormattedString {
-	return IEC.Uint64(s)
-}
-
 // FormattedString represents a human readable representation of a value. It
 // implements the redact.SafeValue interface to signal that it represents a
 // a string that does not need to be redacted.

vendor/github.com/cockroachdb/pebble/internal/invalidating/iter.go

@@ -0,0 +1,162 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package invalidating
+
+import (
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/fastrand"
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// MaybeWrapIfInvariants wraps some iterators with an invalidating iterator.
+// MaybeWrapIfInvariants does nothing in non-invariant builds.
+func MaybeWrapIfInvariants(iter base.InternalIterator) base.InternalIterator {
+	if invariants.Enabled {
+		if fastrand.Uint32n(10) == 1 {
+			return NewIter(iter)
+		}
+	}
+	return iter
+}
+
+// iter tests unsafe key/value slice reuse by modifying the last
+// returned key/value to all 1s.
+type iter struct {
+	iter        base.InternalIterator
+	lastKey     *base.InternalKey
+	lastValue   base.LazyValue
+	ignoreKinds [base.InternalKeyKindMax + 1]bool
+	err         error
+}
+
+// Option configures the behavior of an invalidating iterator.
+type Option interface {
+	apply(*iter)
+}
+
+type funcOpt func(*iter)
+
+func (f funcOpt) apply(i *iter) { f(i) }
+
+// IgnoreKinds constructs an Option that configures an invalidating iterator to
+// skip trashing k/v pairs with the provided key kinds. Some iterators provided
+// key stability guarantees for specific key kinds.
+func IgnoreKinds(kinds ...base.InternalKeyKind) Option {
+	return funcOpt(func(i *iter) {
+		for _, kind := range kinds {
+			i.ignoreKinds[kind] = true
+		}
+	})
+}
+
+// NewIter constructs a new invalidating iterator that wraps the provided
+// iterator, trashing buffers for previously returned keys.
+func NewIter(originalIterator base.InternalIterator, opts ...Option) base.InternalIterator {
+	i := &iter{iter: originalIterator}
+	for _, opt := range opts {
+		opt.apply(i)
+	}
+	return i
+}
+
+func (i *iter) update(
+	key *base.InternalKey, value base.LazyValue,
+) (*base.InternalKey, base.LazyValue) {
+	i.trashLastKV()
+	if key == nil {
+		i.lastKey = nil
+		i.lastValue = base.LazyValue{}
+		return nil, base.LazyValue{}
+	}
+
+	i.lastKey = &base.InternalKey{}
+	*i.lastKey = key.Clone()
+	i.lastValue = base.LazyValue{
+		ValueOrHandle: append(make([]byte, 0, len(value.ValueOrHandle)), value.ValueOrHandle...),
+	}
+	if value.Fetcher != nil {
+		fetcher := new(base.LazyFetcher)
+		*fetcher = *value.Fetcher
+		i.lastValue.Fetcher = fetcher
+	}
+	return i.lastKey, i.lastValue
+}
+
+func (i *iter) trashLastKV() {
+	if i.lastKey == nil {
+		return
+	}
+	if i.ignoreKinds[i.lastKey.Kind()] {
+		return
+	}
+
+	if i.lastKey != nil {
+		for j := range i.lastKey.UserKey {
+			i.lastKey.UserKey[j] = 0xff
+		}
+		i.lastKey.Trailer = 0xffffffffffffffff
+	}
+	for j := range i.lastValue.ValueOrHandle {
+		i.lastValue.ValueOrHandle[j] = 0xff
+	}
+	if i.lastValue.Fetcher != nil {
+		// Not all the LazyFetcher fields are visible, so we zero out the last
+		// value's Fetcher struct entirely.
+		*i.lastValue.Fetcher = base.LazyFetcher{}
+	}
+}
+
+func (i *iter) SeekGE(key []byte, flags base.SeekGEFlags) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekGE(key, flags))
+}
+
+func (i *iter) SeekPrefixGE(
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekPrefixGE(prefix, key, flags))
+}
+
+func (i *iter) SeekLT(key []byte, flags base.SeekLTFlags) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekLT(key, flags))
+}
+
+func (i *iter) First() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.First())
+}
+
+func (i *iter) Last() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Last())
+}
+
+func (i *iter) Next() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Next())
+}
+
+func (i *iter) Prev() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Prev())
+}
+
+func (i *iter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.NextPrefix(succKey))
+}
+
+func (i *iter) Error() error {
+	if err := i.iter.Error(); err != nil {
+		return err
+	}
+	return i.err
+}
+
+func (i *iter) Close() error {
+	return i.iter.Close()
+}
+
+func (i *iter) SetBounds(lower, upper []byte) {
+	i.iter.SetBounds(lower, upper)
+}
+
+func (i *iter) String() string {
+	return i.iter.String()
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go

@@ -228,6 +228,14 @@ func (i *DefragmentingIter) SeekGE(key []byte) *Span {
 	// fragments before this span. Defragment backward to find the start of the
 	// defragmented span.
 	i.defragmentBackward()
+
+	// Defragmenting backward may have stopped because it encountered an error.
+	// If so, we must not continue so that i.iter.Error() (and thus i.Error())
+	// yields the error.
+	if i.iterSpan == nil && i.iter.Error() != nil {
+		return nil
+	}
+
 	if i.iterPos == iterPosPrev {
 		// Next once back onto the span.
 		i.iterSpan = i.iter.Next()
@@ -259,6 +267,14 @@ func (i *DefragmentingIter) SeekLT(key []byte) *Span {
 	// fragments after this span. Defragment forward to find the end of the
 	// defragmented span.
 	i.defragmentForward()
+
+	// Defragmenting forward may have stopped because it encountered an error.
+	// If so, we must not continue so that i.iter.Error() (and thus i.Error())
+	// yields the error.
+	if i.iterSpan == nil && i.iter.Error() != nil {
+		return nil
+	}
+
 	if i.iterPos == iterPosNext {
 		// Prev once back onto the span.
 		i.iterSpan = i.iter.Prev()
@@ -304,7 +320,7 @@ func (i *DefragmentingIter) Next() *Span {
 		// Next once to move onto y, defragment forward to land on the first z
 		// position.
 		i.iterSpan = i.iter.Next()
-		if invariants.Enabled && i.iterSpan == nil {
+		if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
 			panic("pebble: invariant violation: no next span while switching directions")
 		}
 		// We're now positioned on the first span that was defragmented into the
@@ -380,7 +396,7 @@ func (i *DefragmentingIter) Prev() *Span {
 		// Prev once to move onto y, defragment backward to land on the last x
 		// position.
 		i.iterSpan = i.iter.Prev()
-		if invariants.Enabled && i.iterSpan == nil {
+		if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
 			panic("pebble: invariant violation: no previous span while switching directions")
 		}
 		// We're now positioned on the last span that was defragmented into the
@@ -437,6 +453,15 @@ func (i *DefragmentingIter) defragmentForward() *Span {
 		i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
 		i.iterSpan = i.iter.Next()
 	}
+	// i.iterSpan == nil
+	//
+	// The inner iterator may return nil when it encounters an error. If there
+	// was an error, we don't know whether there is another span we should
+	// defragment or not. Return nil so that the caller knows they should check
+	// Error().
+	if i.iter.Error() != nil {
+		return nil
+	}
 	i.curr.Keys = i.keysBuf
 	return &i.curr
 }
@@ -470,6 +495,15 @@ func (i *DefragmentingIter) defragmentBackward() *Span {
 		i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
 		i.iterSpan = i.iter.Prev()
 	}
+	// i.iterSpan == nil
+	//
+	// The inner iterator may return nil when it encounters an error. If there
+	// was an error, we don't know whether there is another span we should
+	// defragment or not. Return nil so that the caller knows they should check
+	// Error().
+	if i.iter.Error() != nil {
+		return nil
+	}
 	i.curr.Keys = i.keysBuf
 	return &i.curr
 }

vendor/github.com/cockroachdb/pebble/internal/keyspan/filter.go

@@ -4,6 +4,8 @@
 
 package keyspan
 
+import "github.com/cockroachdb/pebble/internal/base"
+
 // FilterFunc defines a transform from the input Span into the output Span. The
 // function returns true if the Span should be returned by the iterator, and
 // false if the Span should be skipped. The FilterFunc is permitted to mutate
@@ -21,6 +23,7 @@ type FilterFunc func(in *Span, out *Span) (keep bool)
 type filteringIter struct {
 	iter     FragmentIterator
 	filterFn FilterFunc
+	cmp      base.Compare
 
 	// span is a mutable Span passed to the filterFn. The filterFn is free to
 	// mutate this Span. The slice of Keys in the Span is reused with every call
@@ -32,18 +35,32 @@ var _ FragmentIterator = (*filteringIter)(nil)
 
 // Filter returns a new filteringIter that will filter the Spans from the
 // provided child iterator using the provided FilterFunc.
-func Filter(iter FragmentIterator, filter FilterFunc) FragmentIterator {
+func Filter(iter FragmentIterator, filter FilterFunc, cmp base.Compare) FragmentIterator {
-	return &filteringIter{iter: iter, filterFn: filter}
+	return &filteringIter{iter: iter, filterFn: filter, cmp: cmp}
 }
 
 // SeekGE implements FragmentIterator.
 func (i *filteringIter) SeekGE(key []byte) *Span {
-	return i.filter(i.iter.SeekGE(key), +1)
+	span := i.filter(i.iter.SeekGE(key), +1)
+	// i.filter could return a span that's less than key, _if_ the filterFunc
+	// (which has no knowledge of the seek key) mutated the span to end at a key
+	// less than or equal to `key`. Detect this case and next/invalidate the iter.
+	if span != nil && i.cmp(span.End, key) <= 0 {
+		return i.Next()
+	}
+	return span
 }
 
 // SeekLT implements FragmentIterator.
 func (i *filteringIter) SeekLT(key []byte) *Span {
-	return i.filter(i.iter.SeekLT(key), -1)
+	span := i.filter(i.iter.SeekLT(key), -1)
+	// i.filter could return a span that's >= key, _if_ the filterFunc (which has
+	// no knowledge of the seek key) mutated the span to start at a key greater
+	// than or equal to `key`. Detect this case and prev/invalidate the iter.
+	if span != nil && i.cmp(span.Start, key) >= 0 {
+		return i.Prev()
+	}
+	return span
 }
 
 // First implements FragmentIterator.

vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go

@@ -51,6 +51,8 @@ type FragmentIterator interface {
 	Prev() *Span
 
 	// Error returns any accumulated error.
+	//
+	// TODO(jackson): Lift errors into return values on the positioning methods.
 	Error() error
 
 	// Close closes the iterator and returns any accumulated error. Exhausting
@@ -62,7 +64,7 @@ type FragmentIterator interface {
 
 // TableNewSpanIter creates a new iterator for range key spans for the given
 // file.
-type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions *SpanIterOptions) (FragmentIterator, error)
+type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions SpanIterOptions) (FragmentIterator, error)
 
 // SpanIterOptions is a subset of IterOptions that are necessary to instantiate
 // per-sstable span iterators.

vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go

@@ -98,7 +98,7 @@ func (l *LevelIter) Init(
 ) {
 	l.err = nil
 	l.level = level
-	l.tableOpts.RangeKeyFilters = opts.RangeKeyFilters
+	l.tableOpts = opts
 	l.cmp = cmp
 	l.iterFile = nil
 	l.newIter = newIter
@@ -179,7 +179,7 @@ func (l *LevelIter) loadFile(file *manifest.FileMetadata, dir int) loadFileRetur
 		return noFileLoaded
 	}
 	if indicator != fileAlreadyLoaded {
-		l.iter, l.err = l.newIter(file, &l.tableOpts)
+		l.iter, l.err = l.newIter(file, l.tableOpts)
 		indicator = newFileLoaded
 	}
 	if l.err != nil {

vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go

@@ -25,49 +25,6 @@ import (
 // MergingIter implementation, but will require a bit of plumbing to thread the
 // Equal function.
 
-// Transformer defines a transformation to be applied to a Span.
-type Transformer interface {
-	// Transform takes a Span as input and writes the transformed Span to the
-	// provided output *Span pointer. The output Span's Keys slice may be reused
-	// by Transform to reduce allocations.
-	Transform(cmp base.Compare, in Span, out *Span) error
-}
-
-// The TransformerFunc type is an adapter to allow the use of ordinary functions
-// as Transformers. If f is a function with the appropriate signature,
-// TransformerFunc(f) is a Transformer that calls f.
-type TransformerFunc func(base.Compare, Span, *Span) error
-
-// Transform calls f(cmp, in, out).
-func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
-	return tf(cmp, in, out)
-}
-
-var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
-	dst.Start, dst.End = s.Start, s.End
-	dst.Keys = append(dst.Keys[:0], s.Keys...)
-	return nil
-})
-
-// VisibleTransform filters keys that are invisible at the provided snapshot
-// sequence number.
-func VisibleTransform(snapshot uint64) Transformer {
-	return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
-		dst.Start, dst.End = s.Start, s.End
-		dst.Keys = dst.Keys[:0]
-		for _, k := range s.Keys {
-			// NB: The InternalKeySeqNumMax value is used for the batch snapshot
-			// because a batch's visible span keys are filtered when they're
-			// fragmented. There's no requirement to enforce visibility at
-			// iteration time.
-			if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
-				dst.Keys = append(dst.Keys, k)
-			}
-		}
-		return nil
-	})
-}
-
 // MergingIter merges spans across levels of the LSM, exposing an iterator over
 // spans that yields sets of spans fragmented at unique user key boundaries.
 //

vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go

@@ -0,0 +1,50 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package keyspan
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// Transformer defines a transformation to be applied to a Span.
+type Transformer interface {
+	// Transform takes a Span as input and writes the transformed Span to the
+	// provided output *Span pointer. The output Span's Keys slice may be reused
+	// by Transform to reduce allocations.
+	Transform(cmp base.Compare, in Span, out *Span) error
+}
+
+// The TransformerFunc type is an adapter to allow the use of ordinary functions
+// as Transformers. If f is a function with the appropriate signature,
+// TransformerFunc(f) is a Transformer that calls f.
+type TransformerFunc func(base.Compare, Span, *Span) error
+
+// Transform calls f(cmp, in, out).
+func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error {
+	return tf(cmp, in, out)
+}
+
+var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
+	dst.Start, dst.End = s.Start, s.End
+	dst.Keys = append(dst.Keys[:0], s.Keys...)
+	return nil
+})
+
+// VisibleTransform filters keys that are invisible at the provided snapshot
+// sequence number.
+func VisibleTransform(snapshot uint64) Transformer {
+	return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error {
+		dst.Start, dst.End = s.Start, s.End
+		dst.Keys = dst.Keys[:0]
+		for _, k := range s.Keys {
+			// NB: The InternalKeySeqNumMax value is used for the batch snapshot
+			// because a batch's visible span keys are filtered when they're
+			// fragmented. There's no requirement to enforce visibility at
+			// iteration time.
+			if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) {
+				dst.Keys = append(dst.Keys, k)
+			}
+		}
+		return nil
+	})
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go

@@ -10,13 +10,18 @@ import "github.com/cockroachdb/pebble/internal/base"
 // truncated to be contained within the range [lower, upper). If start and end
 // are specified, filter out any spans that are completely outside those bounds.
 func Truncate(
-	cmp base.Compare, iter FragmentIterator, lower, upper []byte, start, end *base.InternalKey,
+	cmp base.Compare,
+	iter FragmentIterator,
+	lower, upper []byte,
+	start, end *base.InternalKey,
+	panicOnUpperTruncate bool,
 ) FragmentIterator {
 	return Filter(iter, func(in *Span, out *Span) (keep bool) {
 		out.Start, out.End = in.Start, in.End
 		out.Keys = append(out.Keys[:0], in.Keys...)
 
-		// Ignore this span if it lies completely outside [start, end].
+		// Ignore this span if it lies completely outside start, end. Note that
+		// end endInclusive indicated whether end is inclusive.
 		//
 		// The comparison between s.End and start is by user key only, as
 		// the span is exclusive at s.End, so comparing by user keys
@@ -48,13 +53,21 @@ func Truncate(
 				// Wholly within the end bound. Keep it.
 			}
 		}
+
+		var truncated bool
 		// Truncate the bounds to lower and upper.
 		if cmp(in.Start, lower) < 0 {
 			out.Start = lower
 		}
 		if cmp(in.End, upper) > 0 {
+			truncated = true
 			out.End = upper
 		}
+
+		if panicOnUpperTruncate && truncated {
+			panic("pebble: upper bound should not be truncated")
+		}
+
 		return !out.Empty() && cmp(out.Start, out.End) < 0
-	})
+	}, cmp)
 }

vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go

@@ -108,7 +108,7 @@ type annotation struct {
 }
 
 type leafNode struct {
-	ref   int32
+	ref   atomic.Int32
 	count int16
 	leaf  bool
 	// subtreeCount holds the count of files in the entire subtree formed by
@@ -139,13 +139,13 @@ func leafToNode(ln *leafNode) *node {
 func newLeafNode() *node {
 	n := leafToNode(new(leafNode))
 	n.leaf = true
-	n.ref = 1
+	n.ref.Store(1)
 	return n
 }
 
 func newNode() *node {
 	n := new(node)
-	n.ref = 1
+	n.ref.Store(1)
 	return n
 }
 
@@ -159,7 +159,7 @@ func newNode() *node {
 // When a node is cloned, the provided pointer will be redirected to the new
 // mutable node.
 func mut(n **node) *node {
-	if atomic.LoadInt32(&(*n).ref) == 1 {
+	if (*n).ref.Load() == 1 {
 		// Exclusive ownership. Can mutate in place.
 
 		// Whenever a node will be mutated, reset its annotations to be marked
@@ -186,7 +186,7 @@ func mut(n **node) *node {
 
 // incRef acquires a reference to the node.
 func (n *node) incRef() {
-	atomic.AddInt32(&n.ref, 1)
+	n.ref.Add(1)
 }
 
 // decRef releases a reference to the node. If requested, the method will unref
@@ -195,8 +195,8 @@ func (n *node) incRef() {
 // new nodes pass contentsToo=false to preserve existing reference counts during
 // operations that should yield a net-zero change to descendant refcounts.
 // When a node is released, its contained files are dereferenced.
-func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
+func (n *node) decRef(contentsToo bool, obsolete *[]*FileBacking) {
-	if atomic.AddInt32(&n.ref, -1) > 0 {
+	if n.ref.Add(-1) > 0 {
 		// Other references remain. Can't free.
 		return
 	}
@@ -207,7 +207,7 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
 	// nodes, and they want to preserve the existing reference count.
 	if contentsToo {
 		for _, f := range n.items[:n.count] {
-			if atomic.AddInt32(&f.Refs, -1) == 0 {
+			if f.Unref() == 0 {
 				// There are two sources of node dereferences: tree mutations
 				// and Version dereferences. Files should only be made obsolete
 				// during Version dereferences, during which `obsolete` will be
@@ -215,7 +215,13 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
 				if obsolete == nil {
 					panic(fmt.Sprintf("file metadata %s dereferenced to zero during tree mutation", f.FileNum))
 				}
-				*obsolete = append(*obsolete, f)
+				// Reference counting is performed on the FileBacking. In the case
+				// of a virtual sstable, this reference counting is performed on
+				// a FileBacking which is shared by every single virtual sstable
+				// with the same backing sstable. If the reference count hits 0,
+				// then we know that the FileBacking won't be required by any
+				// sstable in Pebble, and that the backing sstable can be deleted.
+				*obsolete = append(*obsolete, f.FileBacking)
 			}
 		}
 		if !n.leaf {
@@ -241,7 +247,7 @@ func (n *node) clone() *node {
 	c.subtreeCount = n.subtreeCount
 	// Increase the refcount of each contained item.
 	for _, f := range n.items[:n.count] {
-		atomic.AddInt32(&f.Refs, 1)
+		f.Ref()
 	}
 	if !c.leaf {
 		// Copy children and increase each refcount.
@@ -761,8 +767,8 @@ type btree struct {
 
 // Release dereferences and clears the root node of the btree, removing all
 // items from the btree. In doing so, it decrements contained file counts.
-// It returns a slice of newly obsolete files, if any.
+// It returns a slice of newly obsolete backing files, if any.
-func (t *btree) Release() (obsolete []*FileMetadata) {
+func (t *btree) Release() (obsolete []*FileBacking) {
 	if t.root != nil {
 		t.root.decRef(true /* contentsToo */, &obsolete)
 		t.root = nil
@@ -800,7 +806,7 @@ func (t *btree) Delete(item *FileMetadata) (obsolete bool) {
 		return false
 	}
 	if out := mut(&t.root).Remove(t.cmp, item); out != nil {
-		obsolete = atomic.AddInt32(&out.Refs, -1) == 0
+		obsolete = out.Unref() == 0
 	}
 	if invariants.Enabled {
 		t.root.verifyInvariants()
@@ -832,7 +838,7 @@ func (t *btree) Insert(item *FileMetadata) error {
 		newRoot.subtreeCount = t.root.subtreeCount + splitNode.subtreeCount + 1
 		t.root = newRoot
 	}
-	atomic.AddInt32(&item.Refs, 1)
+	item.Ref()
 	err := mut(&t.root).Insert(t.cmp, item)
 	if invariants.Enabled {
 		t.root.verifyInvariants()

vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go

@@ -15,20 +15,28 @@ import (
 // LevelMetadata contains metadata for all of the files within
 // a level of the LSM.
 type LevelMetadata struct {
-	level int
+	level     int
-	tree  btree
+	totalSize uint64
+	// NumVirtual is the number of virtual sstables in the level.
+	NumVirtual uint64
+	// VirtualSize is the size of the virtual sstables in the level.
+	VirtualSize uint64
+	tree        btree
 }
 
 // clone makes a copy of the level metadata, implicitly increasing the ref
 // count of every file contained within lm.
 func (lm *LevelMetadata) clone() LevelMetadata {
 	return LevelMetadata{
-		level: lm.level,
+		level:       lm.level,
-		tree:  lm.tree.Clone(),
+		totalSize:   lm.totalSize,
+		NumVirtual:  lm.NumVirtual,
+		VirtualSize: lm.VirtualSize,
+		tree:        lm.tree.Clone(),
 	}
 }
 
-func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
+func (lm *LevelMetadata) release() (obsolete []*FileBacking) {
 	return lm.tree.Release()
 }
 
@@ -40,6 +48,13 @@ func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetad
 	var lm LevelMetadata
 	lm.level = level
 	lm.tree, _ = makeBTree(bcmp, files)
+	for _, f := range files {
+		lm.totalSize += f.Size
+		if f.Virtual {
+			lm.NumVirtual++
+			lm.VirtualSize += f.Size
+		}
+	}
 	return lm
 }
 
@@ -52,6 +67,27 @@ func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) {
 	return t, newLevelSlice(t.Iter())
 }
 
+func (lm *LevelMetadata) insert(f *FileMetadata) error {
+	if err := lm.tree.Insert(f); err != nil {
+		return err
+	}
+	lm.totalSize += f.Size
+	if f.Virtual {
+		lm.NumVirtual++
+		lm.VirtualSize += f.Size
+	}
+	return nil
+}
+
+func (lm *LevelMetadata) remove(f *FileMetadata) bool {
+	lm.totalSize -= f.Size
+	if f.Virtual {
+		lm.NumVirtual--
+		lm.VirtualSize -= f.Size
+	}
+	return lm.tree.Delete(f)
+}
+
 // Empty indicates whether there are any files in the level.
 func (lm *LevelMetadata) Empty() bool {
 	return lm.tree.Count() == 0
@@ -62,6 +98,11 @@ func (lm *LevelMetadata) Len() int {
 	return lm.tree.Count()
 }
 
+// Size returns the cumulative size of all the files within the level.
+func (lm *LevelMetadata) Size() uint64 {
+	return lm.totalSize
+}
+
 // Iter constructs a LevelIterator over the entire level.
 func (lm *LevelMetadata) Iter() LevelIterator {
 	return LevelIterator{iter: lm.tree.Iter()}
@@ -280,6 +321,32 @@ func (ls *LevelSlice) SizeSum() uint64 {
 	return sum
 }
 
+// NumVirtual returns the number of virtual sstables in the level. Its runtime is
+// linear in the length of the slice.
+func (ls *LevelSlice) NumVirtual() uint64 {
+	var n uint64
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		if f.Virtual {
+			n++
+		}
+	}
+	return n
+}
+
+// VirtualSizeSum returns the sum of the sizes of the virtual sstables in the
+// level.
+func (ls *LevelSlice) VirtualSizeSum() uint64 {
+	var sum uint64
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		if f.Virtual {
+			sum += f.Size
+		}
+	}
+	return sum
+}
+
 // Reslice constructs a new slice backed by the same underlying level, with
 // new start and end positions. Reslice invokes the provided function, passing
 // two LevelIterators: one positioned to i's inclusive start and one

vendor/github.com/cockroachdb/pebble/internal/manifest/version.go

@@ -49,6 +49,9 @@ type TableStats struct {
 	// The number of point and range deletion entries in the table.
 	NumDeletions uint64
 	// NumRangeKeySets is the total number of range key sets in the table.
+	//
+	// NB: If there's a chance that the sstable contains any range key sets,
+	// then NumRangeKeySets must be > 0.
 	NumRangeKeySets uint64
 	// Estimate of the total disk space that may be dropped by this table's
 	// point deletions by compacting them.
@@ -60,9 +63,11 @@ type TableStats struct {
 	// overlapping data in L0 and ignores L0 sublevels, but the error that
 	// introduces is expected to be small.
 	//
-	// Tables in the bottommost level of the LSM may have a nonzero estimate
+	// Tables in the bottommost level of the LSM may have a nonzero estimate if
-	// if snapshots or move compactions prevented the elision of their range
+	// snapshots or move compactions prevented the elision of their range
-	// tombstones.
+	// tombstones. A table in the bottommost level that was ingested into L6
+	// will have a zero estimate, because the file's sequence numbers indicate
+	// that the tombstone cannot drop any data contained within the file itself.
 	RangeDeletionsBytesEstimate uint64
 	// Total size of value blocks and value index block.
 	ValueBlocksSize uint64
@@ -119,43 +124,72 @@ func (s CompactionState) String() string {
 	}
 }
 
-// FileMetadata holds the metadata for an on-disk table.
+// FileMetadata is maintained for leveled-ssts, i.e., they belong to a level of
+// some version. FileMetadata does not contain the actual level of the sst,
+// since such leveled-ssts can move across levels in different versions, while
+// sharing the same FileMetadata. There are two kinds of leveled-ssts, physical
+// and virtual. Underlying both leveled-ssts is a backing-sst, for which the
+// only state is FileBacking. A backing-sst is level-less. It is possible for a
+// backing-sst to be referred to by a physical sst in one version and by one or
+// more virtual ssts in one or more versions. A backing-sst becomes obsolete
+// and can be deleted once it is no longer required by any physical or virtual
+// sst in any version.
+//
+// We maintain some invariants:
+//
+//  1. Each physical and virtual sst will have a unique FileMetadata.FileNum,
+//     and there will be exactly one FileMetadata associated with the FileNum.
+//
+//  2. Within a version, a backing-sst is either only referred to by one
+//     physical sst or one or more virtual ssts.
+//
+//  3. Once a backing-sst is referred to by a virtual sst in the latest version,
+//     it cannot go back to being referred to by a physical sst in any future
+//     version.
+//
+// Once a physical sst is no longer needed by any version, we will no longer
+// maintain the file metadata associated with it. We will still maintain the
+// FileBacking associated with the physical sst if the backing sst is required
+// by any virtual ssts in any version.
 type FileMetadata struct {
-	// Atomic contains fields which are accessed atomically. Go allocations
+	// AllowedSeeks is used to determine if a file should be picked for
-	// are guaranteed to be 64-bit aligned which we take advantage of by
+	// a read triggered compaction. It is decremented when read sampling
-	// placing the 64-bit fields which we access atomically at the beginning
+	// in pebble.Iterator after every after every positioning operation
-	// of the FileMetadata struct. For more information, see
+	// that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc).
-	// https://golang.org/pkg/sync/atomic/#pkg-note-BUG.
+	AllowedSeeks atomic.Int64
-	Atomic struct {
-		// AllowedSeeks is used to determine if a file should be picked for
-		// a read triggered compaction. It is decremented when read sampling
-		// in pebble.Iterator after every after every positioning operation
-		// that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc).
-		AllowedSeeks int64
 
-		// statsValid is 1 if stats have been loaded for the table. The
+	// statsValid indicates if stats have been loaded for the table. The
-		// TableStats structure is populated only if valid is 1.
+	// TableStats structure is populated only if valid is true.
-		statsValid uint32
+	statsValid atomic.Bool
-	}
+
+	// FileBacking is the state which backs either a physical or virtual
+	// sstables.
+	FileBacking *FileBacking
 
 	// InitAllowedSeeks is the inital value of allowed seeks. This is used
 	// to re-set allowed seeks on a file once it hits 0.
 	InitAllowedSeeks int64
-
-	// Reference count for the file: incremented when a file is added to a
-	// version and decremented when the version is unreferenced. The file is
-	// obsolete when the reference count falls to zero.
-	Refs int32
 	// FileNum is the file number.
+	//
+	// INVARIANT: when !FileMetadata.Virtual, FileNum == FileBacking.DiskFileNum.
 	FileNum base.FileNum
-	// Size is the size of the file, in bytes.
+	// Size is the size of the file, in bytes. Size is an approximate value for
+	// virtual sstables.
+	//
+	// INVARIANTS:
+	// - When !FileMetadata.Virtual, Size == FileBacking.Size.
+	// - Size should be non-zero. Size 0 virtual sstables must not be created.
 	Size uint64
 	// File creation time in seconds since the epoch (1970-01-01 00:00:00
 	// UTC). For ingested sstables, this corresponds to the time the file was
-	// ingested.
+	// ingested. For virtual sstables, this corresponds to the wall clock time
+	// when the FileMetadata for the virtual sstable was first created.
 	CreationTime int64
-	// Smallest and largest sequence numbers in the table, across both point and
+	// Lower and upper bounds for the smallest and largest sequence numbers in
-	// range keys.
+	// the table, across both point and range keys. For physical sstables, these
+	// values are tight bounds. For virtual sstables, there is no guarantee that
+	// there will be keys with SmallestSeqNum or LargestSeqNum within virtual
+	// sstable bounds.
 	SmallestSeqNum uint64
 	LargestSeqNum  uint64
 	// SmallestPointKey and LargestPointKey are the inclusive bounds for the
@@ -178,8 +212,17 @@ type FileMetadata struct {
 	Smallest InternalKey
 	Largest  InternalKey
 	// Stats describe table statistics. Protected by DB.mu.
+	//
+	// For virtual sstables, set stats upon virtual sstable creation as
+	// asynchronous computation of stats is not currently supported.
+	//
+	// TODO(bananabrick): To support manifest replay for virtual sstables, we
+	// probably need to compute virtual sstable stats asynchronously. Otherwise,
+	// we'd have to write virtual sstable stats to the version edit.
 	Stats TableStats
 
+	// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
+	// pick L0 compactions. Only accurate for the most recent Version.
 	SubLevel         int
 	L0Index          int
 	minIntervalIndex int
@@ -188,9 +231,6 @@ type FileMetadata struct {
 	// NB: the alignment of this struct is 8 bytes. We pack all the bools to
 	// ensure an optimal packing.
 
-	// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
-	// pick L0 compactions. Only accurate for the most recent Version.
-	//
 	// IsIntraL0Compacting is set to True if this file is part of an intra-L0
 	// compaction. When it's true, IsCompacting must also return true. If
 	// Compacting is true and IsIntraL0Compacting is false for an L0 file, the
@@ -226,6 +266,212 @@ type FileMetadata struct {
 	// key type (point or range) corresponds to the smallest and largest overall
 	// table bounds.
 	boundTypeSmallest, boundTypeLargest boundType
+	// Virtual is true if the FileMetadata belongs to a virtual sstable.
+	Virtual bool
+}
+
+// PhysicalFileMeta is used by functions which want a guarantee that their input
+// belongs to a physical sst and not a virtual sst.
+//
+// NB: This type should only be constructed by calling
+// FileMetadata.PhysicalMeta.
+type PhysicalFileMeta struct {
+	*FileMetadata
+}
+
+// VirtualFileMeta is used by functions which want a guarantee that their input
+// belongs to a virtual sst and not a physical sst.
+//
+// A VirtualFileMeta inherits all the same fields as a FileMetadata. These
+// fields have additional invariants imposed on them, and/or slightly varying
+// meanings:
+//   - Smallest and Largest (and their counterparts
+//     {Smallest, Largest}{Point,Range}Key) remain tight bounds that represent a
+//     key at that exact bound. We make the effort to determine the next smallest
+//     or largest key in an sstable after virtualizing it, to maintain this
+//     tightness. If the largest is a sentinel key (IsExclusiveSentinel()), it
+//     could mean that a rangedel or range key ends at that user key, or has been
+//     truncated to that user key.
+//   - One invariant is that if a rangedel or range key is truncated on its
+//     upper bound, the virtual sstable *must* have a rangedel or range key
+//     sentinel key as its upper bound. This is because truncation yields
+//     an exclusive upper bound for the rangedel/rangekey, and if there are
+//     any points at that exclusive upper bound within the same virtual
+//     sstable, those could get uncovered by this truncation. We enforce this
+//     invariant in calls to keyspan.Truncate.
+//   - Size is an estimate of the size of the virtualized portion of this sstable.
+//     The underlying file's size is stored in FileBacking.Size, though it could
+//     also be estimated or could correspond to just the referenced portion of
+//     a file (eg. if the file originated on another node).
+//   - Size must be > 0.
+//   - SmallestSeqNum and LargestSeqNum are loose bounds for virtual sstables.
+//     This means that all keys in the virtual sstable must have seqnums within
+//     [SmallestSeqNum, LargestSeqNum], however there's no guarantee that there's
+//     a key with a seqnum at either of the bounds. Calculating tight seqnum
+//     bounds would be too expensive and deliver little value.
+//
+// NB: This type should only be constructed by calling FileMetadata.VirtualMeta.
+type VirtualFileMeta struct {
+	*FileMetadata
+}
+
+// PhysicalMeta should be the only source of creating the PhysicalFileMeta
+// wrapper type.
+func (m *FileMetadata) PhysicalMeta() PhysicalFileMeta {
+	if m.Virtual {
+		panic("pebble: file metadata does not belong to a physical sstable")
+	}
+	return PhysicalFileMeta{
+		m,
+	}
+}
+
+// VirtualMeta should be the only source of creating the VirtualFileMeta wrapper
+// type.
+func (m *FileMetadata) VirtualMeta() VirtualFileMeta {
+	if !m.Virtual {
+		panic("pebble: file metadata does not belong to a virtual sstable")
+	}
+	return VirtualFileMeta{
+		m,
+	}
+}
+
+// FileBacking either backs a single physical sstable, or one or more virtual
+// sstables.
+//
+// See the comment above the FileMetadata type for sstable terminology.
+type FileBacking struct {
+	// Reference count for the backing file on disk: incremented when a
+	// physical or virtual sstable which is backed by the FileBacking is
+	// added to a version and decremented when the version is unreferenced.
+	// We ref count in order to determine when it is safe to delete a
+	// backing sst file from disk. The backing file is obsolete when the
+	// reference count falls to zero.
+	refs atomic.Int32
+	// latestVersionRefs are the references to the FileBacking in the
+	// latest version. This reference can be through a single physical
+	// sstable in the latest version, or one or more virtual sstables in the
+	// latest version.
+	//
+	// INVARIANT: latestVersionRefs <= refs.
+	latestVersionRefs atomic.Int32
+	// VirtualizedSize is set iff the backing sst is only referred to by
+	// virtual ssts in the latest version. VirtualizedSize is the sum of the
+	// virtual sstable sizes of all of the virtual sstables in the latest
+	// version which are backed by the physical sstable. When a virtual
+	// sstable is removed from the latest version, we will decrement the
+	// VirtualizedSize. During compaction picking, we'll compensate a
+	// virtual sstable file size by
+	// (FileBacking.Size - FileBacking.VirtualizedSize) / latestVersionRefs.
+	// The intuition is that if FileBacking.Size - FileBacking.VirtualizedSize
+	// is high, then the space amplification due to virtual sstables is
+	// high, and we should pick the virtual sstable with a higher priority.
+	//
+	// TODO(bananabrick): Compensate the virtual sstable file size using
+	// the VirtualizedSize during compaction picking and test.
+	VirtualizedSize atomic.Uint64
+	DiskFileNum     base.DiskFileNum
+	Size            uint64
+}
+
+// InitPhysicalBacking allocates and sets the FileBacking which is required by a
+// physical sstable FileMetadata.
+//
+// Ensure that the state required by FileBacking, such as the FileNum, is
+// already set on the FileMetadata before InitPhysicalBacking is called.
+// Calling InitPhysicalBacking only after the relevant state has been set in the
+// FileMetadata is not necessary in tests which don't rely on FileBacking.
+func (m *FileMetadata) InitPhysicalBacking() {
+	if m.Virtual {
+		panic("pebble: virtual sstables should use a pre-existing FileBacking")
+	}
+	if m.FileBacking == nil {
+		m.FileBacking = &FileBacking{Size: m.Size, DiskFileNum: m.FileNum.DiskFileNum()}
+	}
+}
+
+// InitProviderBacking creates a new FileBacking for a file backed by
+// an objstorage.Provider.
+func (m *FileMetadata) InitProviderBacking(fileNum base.DiskFileNum) {
+	if !m.Virtual {
+		panic("pebble: provider-backed sstables must be virtual")
+	}
+	if m.FileBacking == nil {
+		m.FileBacking = &FileBacking{DiskFileNum: fileNum}
+	}
+}
+
+// ValidateVirtual should be called once the FileMetadata for a virtual sstable
+// is created to verify that the fields of the virtual sstable are sound.
+func (m *FileMetadata) ValidateVirtual(createdFrom *FileMetadata) {
+	if !m.Virtual {
+		panic("pebble: invalid virtual sstable")
+	}
+
+	if createdFrom.SmallestSeqNum != m.SmallestSeqNum {
+		panic("pebble: invalid smallest sequence number for virtual sstable")
+	}
+
+	if createdFrom.LargestSeqNum != m.LargestSeqNum {
+		panic("pebble: invalid largest sequence number for virtual sstable")
+	}
+
+	if createdFrom.FileBacking != nil && createdFrom.FileBacking != m.FileBacking {
+		panic("pebble: invalid physical sstable state for virtual sstable")
+	}
+
+	if m.Size == 0 {
+		panic("pebble: virtual sstable size must be set upon creation")
+	}
+}
+
+// Refs returns the refcount of backing sstable.
+func (m *FileMetadata) Refs() int32 {
+	return m.FileBacking.refs.Load()
+}
+
+// Ref increments the ref count associated with the backing sstable.
+func (m *FileMetadata) Ref() {
+	m.FileBacking.refs.Add(1)
+}
+
+// Unref decrements the ref count associated with the backing sstable.
+func (m *FileMetadata) Unref() int32 {
+	v := m.FileBacking.refs.Add(-1)
+	if invariants.Enabled && v < 0 {
+		panic("pebble: invalid FileMetadata refcounting")
+	}
+	return v
+}
+
+// LatestRef increments the latest ref count associated with the backing
+// sstable.
+func (m *FileMetadata) LatestRef() {
+	m.FileBacking.latestVersionRefs.Add(1)
+
+	if m.Virtual {
+		m.FileBacking.VirtualizedSize.Add(m.Size)
+	}
+}
+
+// LatestUnref decrements the latest ref count associated with the backing
+// sstable.
+func (m *FileMetadata) LatestUnref() int32 {
+	if m.Virtual {
+		m.FileBacking.VirtualizedSize.Add(-m.Size)
+	}
+
+	v := m.FileBacking.latestVersionRefs.Add(-1)
+	if invariants.Enabled && v < 0 {
+		panic("pebble: invalid FileMetadata latest refcounting")
+	}
+	return v
+}
+
+// LatestRefs returns the latest ref count associated with the backing sstable.
+func (m *FileMetadata) LatestRefs() int32 {
+	return m.FileBacking.latestVersionRefs.Load()
 }
 
 // SetCompactionState transitions this file's compaction state to the given
@@ -263,22 +509,14 @@ func (m *FileMetadata) IsCompacting() bool {
 // returns true, the Stats field may be read (with or without holding the
 // database mutex).
 func (m *FileMetadata) StatsValid() bool {
-	return atomic.LoadUint32(&m.Atomic.statsValid) == 1
+	return m.statsValid.Load()
-}
-
-// StatsValidLocked returns true if the table stats have been populated.
-// StatsValidLocked requires DB.mu is held when it's invoked, and it avoids the
-// overhead of an atomic load. This is possible because table stats validity is
-// only set while DB.mu is held.
-func (m *FileMetadata) StatsValidLocked() bool {
-	return m.Atomic.statsValid == 1
 }
 
 // StatsMarkValid marks the TableStats as valid. The caller must hold DB.mu
 // while populating TableStats and calling StatsMarkValud. Once stats are
 // populated, they must not be mutated.
 func (m *FileMetadata) StatsMarkValid() {
-	atomic.StoreUint32(&m.Atomic.statsValid, 1)
+	m.statsValid.Store(true)
 }
 
 // ExtendPointKeyBounds attempts to extend the lower and upper point key bounds
@@ -369,6 +607,13 @@ func (m *FileMetadata) Overlaps(cmp Compare, start []byte, end []byte, exclusive
 	return true
 }
 
+// ContainedWithinSpan returns true if the file key range completely overlaps with the
+// given range ("end" is assumed to exclusive).
+func (m *FileMetadata) ContainedWithinSpan(cmp Compare, start, end []byte) bool {
+	lowerCmp, upperCmp := cmp(m.Smallest.UserKey, start), cmp(m.Largest.UserKey, end)
+	return lowerCmp >= 0 && (upperCmp < 0 || (upperCmp == 0 && m.Largest.IsExclusiveSentinel()))
+}
+
 // ContainsKeyType returns whether or not the file contains keys of the provided
 // type.
 func (m *FileMetadata) ContainsKeyType(kt KeyType) bool {
@@ -466,6 +711,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
 	if !verbose {
 		return b.String()
 	}
+	fmt.Fprintf(&b, " seqnums:[%d-%d]", m.SmallestSeqNum, m.LargestSeqNum)
 	if m.HasPointKeys {
 		fmt.Fprintf(&b, " points:[%s-%s]",
 			m.SmallestPointKey.Pretty(format), m.LargestPointKey.Pretty(format))
@@ -479,9 +725,9 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
 
 // ParseFileMetadataDebug parses a FileMetadata from its DebugString
 // representation.
-func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
+func ParseFileMetadataDebug(s string) (*FileMetadata, error) {
 	// Split lines of the form:
-	//  000000:[a#0,SET-z#0,SET] points:[...] ranges:[...]
+	//  000000:[a#0,SET-z#0,SET] seqnums:[5-5] points:[...] ranges:[...]
 	fields := strings.FieldsFunc(s, func(c rune) bool {
 		switch c {
 		case ':', '[', '-', ']':
@@ -491,10 +737,24 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
 		}
 	})
 	if len(fields)%3 != 0 {
-		return m, errors.Newf("malformed input: %s", s)
+		return nil, errors.Newf("malformed input: %s", s)
 	}
+	m := &FileMetadata{}
 	for len(fields) > 0 {
 		prefix := fields[0]
+		if prefix == "seqnums" {
+			smallestSeqNum, err := strconv.ParseUint(fields[1], 10, 64)
+			if err != nil {
+				return m, errors.Newf("malformed input: %s: %s", s, err)
+			}
+			largestSeqNum, err := strconv.ParseUint(fields[2], 10, 64)
+			if err != nil {
+				return m, errors.Newf("malformed input: %s: %s", s, err)
+			}
+			m.SmallestSeqNum, m.LargestSeqNum = smallestSeqNum, largestSeqNum
+			fields = fields[3:]
+			continue
+		}
 		smallest := base.ParsePrettyInternalKey(fields[1])
 		largest := base.ParsePrettyInternalKey(fields[2])
 		switch prefix {
@@ -522,7 +782,8 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) {
 		m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest
 		m.HasPointKeys = true
 	}
-	return
+	m.InitPhysicalBacking()
+	return m, nil
 }
 
 // Validate validates the metadata for consistency with itself, returning an
@@ -584,6 +845,11 @@ func (m *FileMetadata) Validate(cmp Compare, formatKey base.FormatKey) error {
 		}
 	}
 
+	// Ensure that FileMetadata.Init was called.
+	if m.FileBacking == nil {
+		return base.CorruptionErrorf("file metadata FileBacking not set")
+	}
+
 	return nil
 }
 
@@ -756,6 +1022,9 @@ func NewVersion(
 		} else {
 			v.Levels[l].tree.cmp = btreeCmpSmallestKey(cmp)
 		}
+		for _, f := range files[l] {
+			v.Levels[l].totalSize += f.Size
+		}
 	}
 	if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
 		panic(err)
@@ -788,7 +1057,7 @@ func NewVersion(
 // key in a higher level table that has both the same user key and a higher
 // sequence number.
 type Version struct {
-	refs int32
+	refs atomic.Int32
 
 	// The level 0 sstables are organized in a series of sublevels. Similar to
 	// the seqnum invariant in normal levels, there is no internal key in a
@@ -805,7 +1074,7 @@ type Version struct {
 	// usage.
 	//
 	// L0Sublevels.Levels contains L0 files ordered by sublevels. All the files
-	// in Files[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to
+	// in Levels[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to
 	// a reference to that slice, as that slice is necessary for iterator
 	// creation and needs to outlast L0Sublevels.
 	L0Sublevels     *L0Sublevels
@@ -820,7 +1089,7 @@ type Version struct {
 
 	// The callback to invoke when the last reference to a version is
 	// removed. Will be called with list.mu held.
-	Deleted func(obsolete []*FileMetadata)
+	Deleted func(obsolete []*FileBacking)
 
 	// Stats holds aggregated stats about the version maintained from
 	// version to version.
@@ -849,15 +1118,21 @@ func (v *Version) DebugString(format base.FormatKey) string {
 	return v.string(format, true)
 }
 
+func describeSublevels(format base.FormatKey, verbose bool, sublevels []LevelSlice) string {
+	var buf bytes.Buffer
+	for sublevel := len(sublevels) - 1; sublevel >= 0; sublevel-- {
+		fmt.Fprintf(&buf, "0.%d:\n", sublevel)
+		sublevels[sublevel].Each(func(f *FileMetadata) {
+			fmt.Fprintf(&buf, "  %s\n", f.DebugString(format, verbose))
+		})
+	}
+	return buf.String()
+}
+
 func (v *Version) string(format base.FormatKey, verbose bool) string {
 	var buf bytes.Buffer
 	if len(v.L0SublevelFiles) > 0 {
-		for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
+		fmt.Fprintf(&buf, "%s", describeSublevels(format, verbose, v.L0SublevelFiles))
-			fmt.Fprintf(&buf, "0.%d:\n", sublevel)
-			v.L0SublevelFiles[sublevel].Each(func(f *FileMetadata) {
-				fmt.Fprintf(&buf, "  %s\n", f.DebugString(format, verbose))
-			})
-		}
 	}
 	for level := 1; level < NumLevels; level++ {
 		if v.Levels[level].Empty() {
@@ -898,7 +1173,7 @@ func ParseVersionDebug(
 				m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest
 				m.HasPointKeys = true
 			}
-			files[level] = append(files[level], &m)
+			files[level] = append(files[level], m)
 		}
 	}
 	// Reverse the order of L0 files. This ensures we construct the same
@@ -912,12 +1187,12 @@ func ParseVersionDebug(
 
 // Refs returns the number of references to the version.
 func (v *Version) Refs() int32 {
-	return atomic.LoadInt32(&v.refs)
+	return v.refs.Load()
 }
 
 // Ref increments the version refcount.
 func (v *Version) Ref() {
-	atomic.AddInt32(&v.refs, 1)
+	v.refs.Add(1)
 }
 
 // Unref decrements the version refcount. If the last reference to the version
@@ -925,12 +1200,11 @@ func (v *Version) Ref() {
 // Deleted callback is invoked. Requires that the VersionList mutex is NOT
 // locked.
 func (v *Version) Unref() {
-	if atomic.AddInt32(&v.refs, -1) == 0 {
+	if v.refs.Add(-1) == 0 {
-		obsolete := v.unrefFiles()
 		l := v.list
 		l.mu.Lock()
 		l.Remove(v)
-		v.Deleted(obsolete)
+		v.Deleted(v.unrefFiles())
 		l.mu.Unlock()
 	}
 }
@@ -940,14 +1214,14 @@ func (v *Version) Unref() {
 // the Deleted callback is invoked. Requires that the VersionList mutex is
 // already locked.
 func (v *Version) UnrefLocked() {
-	if atomic.AddInt32(&v.refs, -1) == 0 {
+	if v.refs.Add(-1) == 0 {
 		v.list.Remove(v)
 		v.Deleted(v.unrefFiles())
 	}
 }
 
-func (v *Version) unrefFiles() []*FileMetadata {
+func (v *Version) unrefFiles() []*FileBacking {
-	var obsolete []*FileMetadata
+	var obsolete []*FileBacking
 	for _, lm := range v.Levels {
 		obsolete = append(obsolete, lm.release()...)
 	}
@@ -1079,16 +1353,20 @@ func (v *Version) Overlaps(
 // CheckOrdering checks that the files are consistent with respect to
 // increasing file numbers (for level 0 files) and increasing and non-
 // overlapping internal key ranges (for level non-0 files).
-func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error {
+func (v *Version) CheckOrdering(
+	cmp Compare, format base.FormatKey, order OrderingInvariants,
+) error {
 	for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
 		sublevelIter := v.L0SublevelFiles[sublevel].Iter()
-		if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter); err != nil {
+		// Sublevels have NEVER allowed split user keys, so we can pass
+		// ProhibitSplitUserKeys.
+		if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter, ProhibitSplitUserKeys); err != nil {
 			return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
 		}
 	}
 
 	for level, lm := range v.Levels {
-		if err := CheckOrdering(cmp, format, Level(level), lm.Iter()); err != nil {
+		if err := CheckOrdering(cmp, format, Level(level), lm.Iter(), order); err != nil {
 			return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
 		}
 	}
@@ -1157,10 +1435,34 @@ func (l *VersionList) Remove(v *Version) {
 	v.list = nil // avoid memory leaks
 }
 
+// OrderingInvariants dictates the file ordering invariants active.
+type OrderingInvariants int8
+
+const (
+	// ProhibitSplitUserKeys indicates that adjacent files within a level cannot
+	// contain the same user key.
+	ProhibitSplitUserKeys OrderingInvariants = iota
+	// AllowSplitUserKeys indicates that adjacent files within a level may
+	// contain the same user key. This is only allowed by historical format
+	// major versions.
+	//
+	// TODO(jackson): Remove.
+	AllowSplitUserKeys
+)
+
 // CheckOrdering checks that the files are consistent with respect to
 // seqnums (for level 0 files -- see detailed comment below) and increasing and non-
 // overlapping internal key ranges (for non-level 0 files).
-func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelIterator) error {
+//
+// The ordering field may be passed AllowSplitUserKeys to allow adjacent files that are both
+// inclusive of the same user key. Pebble no longer creates version edits
+// installing such files, and Pebble databases with sufficiently high format
+// major version should no longer have any such files within their LSM.
+// TODO(jackson): Remove AllowSplitUserKeys when we remove support for the
+// earlier format major versions.
+func CheckOrdering(
+	cmp Compare, format base.FormatKey, level Level, files LevelIterator, ordering OrderingInvariants,
+) error {
 	// The invariants to check for L0 sublevels are the same as the ones to
 	// check for all other levels. However, if L0 is not organized into
 	// sublevels, or if all L0 files are being passed in, we do the legacy L0
@@ -1238,11 +1540,29 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelI
 						prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
 						f.Smallest.Pretty(format), f.Largest.Pretty(format))
 				}
-				if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
+
-					return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
+				// What's considered "overlapping" is dependent on the format
-						errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
+				// major version. If ordering=ProhibitSplitUserKeys, then both
-						prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
+				// files cannot contain keys with the same user keys. If the
-						f.Smallest.Pretty(format), f.Largest.Pretty(format))
+				// bounds have the same user key, the previous file's boundary
+				// must have a Trailer indicating that it's exclusive.
+				switch ordering {
+				case AllowSplitUserKeys:
+					if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
+						return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
+							errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
+							prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
+							f.Smallest.Pretty(format), f.Largest.Pretty(format))
+					}
+				case ProhibitSplitUserKeys:
+					if v := cmp(prev.Largest.UserKey, f.Smallest.UserKey); v > 0 || (v == 0 && !prev.Largest.IsExclusiveSentinel()) {
+						return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
+							errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
+							prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
+							f.Smallest.Pretty(format), f.Largest.Pretty(format))
+					}
+				default:
+					panic("unreachable")
 				}
 			}
 		}

vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go

@@ -11,11 +11,11 @@ import (
 	"fmt"
 	"io"
 	"sort"
-	"sync/atomic"
 	"time"
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
 )
 
 // TODO(peter): describe the MANIFEST file format, independently of the C++
@@ -51,7 +51,9 @@ const (
 	tagMaxColumnFamily  = 203
 
 	// Pebble tags.
-	tagNewFile5 = 104 // Range keys.
+	tagNewFile5            = 104 // Range keys.
+	tagCreatedBackingTable = 105
+	tagRemovedBackingTable = 106
 
 	// The custom tags sub-format used by tagNewFile4 and above.
 	customTagTerminate         = 1
@@ -59,6 +61,7 @@ const (
 	customTagCreationTime      = 6
 	customTagPathID            = 65
 	customTagNonSafeIgnoreMask = 1 << 6
+	customTagVirtual           = 66
 )
 
 // DeletedFileEntry holds the state for a file deletion from a level. The file
@@ -73,6 +76,9 @@ type DeletedFileEntry struct {
 type NewFileEntry struct {
 	Level int
 	Meta  *FileMetadata
+	// BackingFileNum is only set during manifest replay, and only for virtual
+	// sstables.
+	BackingFileNum base.DiskFileNum
 }
 
 // VersionEdit holds the state for an edit to a Version along with other
@@ -110,9 +116,42 @@ type VersionEdit struct {
 	// found that there was no overlapping file at the higher level).
 	DeletedFiles map[DeletedFileEntry]*FileMetadata
 	NewFiles     []NewFileEntry
+	// CreatedBackingTables can be used to preserve the FileBacking associated
+	// with a physical sstable. This is useful when virtual sstables in the
+	// latest version are reconstructed during manifest replay, and we also need
+	// to reconstruct the FileBacking which is required by these virtual
+	// sstables.
+	//
+	// INVARIANT: The FileBacking associated with a physical sstable must only
+	// be added as a backing file in the same version edit where the physical
+	// sstable is first virtualized. This means that the physical sstable must
+	// be present in DeletedFiles and that there must be at least one virtual
+	// sstable with the same FileBacking as the physical sstable in NewFiles. A
+	// file must be present in CreatedBackingTables in exactly one version edit.
+	// The physical sstable associated with the FileBacking must also not be
+	// present in NewFiles.
+	CreatedBackingTables []*FileBacking
+	// RemovedBackingTables is used to remove the FileBacking associated with a
+	// virtual sstable. Note that a backing sstable can be removed as soon as
+	// there are no virtual sstables in the latest version which are using the
+	// backing sstable, but the backing sstable doesn't necessarily have to be
+	// removed atomically with the version edit which removes the last virtual
+	// sstable associated with the backing sstable. The removal can happen in a
+	// future version edit.
+	//
+	// INVARIANT: A file must only be added to RemovedBackingTables if it was
+	// added to CreateBackingTables in a prior version edit. The same version
+	// edit also cannot have the same file present in both CreateBackingTables
+	// and RemovedBackingTables. A file must be present in RemovedBackingTables
+	// in exactly one version edit.
+	RemovedBackingTables []base.DiskFileNum
 }
 
 // Decode decodes an edit from the specified reader.
+//
+// Note that the Decode step will not set the FileBacking for virtual sstables
+// and the responsibility is left to the caller. However, the Decode step will
+// populate the NewFileEntry.BackingFileNum in VersionEdit.NewFiles.
 func (v *VersionEdit) Decode(r io.Reader) error {
 	br, ok := r.(byteReader)
 	if !ok {
@@ -165,6 +204,28 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 			}
 			// NB: RocksDB does not use compaction pointers anymore.
 
+		case tagRemovedBackingTable:
+			n, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			v.RemovedBackingTables = append(
+				v.RemovedBackingTables, base.FileNum(n).DiskFileNum(),
+			)
+		case tagCreatedBackingTable:
+			dfn, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			size, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			fileBacking := &FileBacking{
+				DiskFileNum: base.FileNum(dfn).DiskFileNum(),
+				Size:        size,
+			}
+			v.CreatedBackingTables = append(v.CreatedBackingTables, fileBacking)
 		case tagDeletedFile:
 			level, err := d.readLevel()
 			if err != nil {
@@ -270,6 +331,10 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 			}
 			var markedForCompaction bool
 			var creationTime uint64
+			virtualState := struct {
+				virtual        bool
+				backingFileNum uint64
+			}{}
 			if tag == tagNewFile4 || tag == tagNewFile5 {
 				for {
 					customTag, err := d.readUvarint()
@@ -278,7 +343,16 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 					}
 					if customTag == customTagTerminate {
 						break
+					} else if customTag == customTagVirtual {
+						virtualState.virtual = true
+						n, err := d.readUvarint()
+						if err != nil {
+							return err
+						}
+						virtualState.backingFileNum = n
+						continue
 					}
+
 					field, err := d.readBytes()
 					if err != nil {
 						return err
@@ -314,6 +388,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 				SmallestSeqNum:      smallestSeqNum,
 				LargestSeqNum:       largestSeqNum,
 				MarkedForCompaction: markedForCompaction,
+				Virtual:             virtualState.virtual,
 			}
 			if tag != tagNewFile5 { // no range keys present
 				m.SmallestPointKey = base.DecodeInternalKey(smallestPointKey)
@@ -345,10 +420,18 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 				}
 			}
 			m.boundsSet = true
-			v.NewFiles = append(v.NewFiles, NewFileEntry{
+			if !virtualState.virtual {
+				m.InitPhysicalBacking()
+			}
+
+			nfe := NewFileEntry{
 				Level: level,
 				Meta:  m,
-			})
+			}
+			if virtualState.virtual {
+				nfe.BackingFileNum = base.FileNum(virtualState.backingFileNum).DiskFileNum()
+			}
+			v.NewFiles = append(v.NewFiles, nfe)
 
 		case tagPrevLogNumber:
 			n, err := d.readUvarint()
@@ -367,8 +450,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 	return nil
 }
 
-// String implements fmt.Stringer for a VersionEdit.
+func (v *VersionEdit) string(verbose bool, fmtKey base.FormatKey) string {
-func (v *VersionEdit) String() string {
 	var buf bytes.Buffer
 	if v.ComparerName != "" {
 		fmt.Fprintf(&buf, "  comparer:     %s", v.ComparerName)
@@ -399,7 +481,12 @@ func (v *VersionEdit) String() string {
 		fmt.Fprintf(&buf, "  deleted:       L%d %s\n", df.Level, df.FileNum)
 	}
 	for _, nf := range v.NewFiles {
-		fmt.Fprintf(&buf, "  added:         L%d %s", nf.Level, nf.Meta.String())
+		fmt.Fprintf(&buf, "  added:         L%d", nf.Level)
+		if verbose {
+			fmt.Fprintf(&buf, " %s", nf.Meta.DebugString(fmtKey, true /* verbose */))
+		} else {
+			fmt.Fprintf(&buf, " %s", nf.Meta.String())
+		}
 		if nf.Meta.CreationTime != 0 {
 			fmt.Fprintf(&buf, " (%s)",
 				time.Unix(nf.Meta.CreationTime, 0).UTC().Format(time.RFC3339))
@@ -409,6 +496,16 @@ func (v *VersionEdit) String() string {
 	return buf.String()
 }
 
+// DebugString is a more verbose version of String(). Use this in tests.
+func (v *VersionEdit) DebugString(fmtKey base.FormatKey) string {
+	return v.string(true /* verbose */, fmtKey)
+}
+
+// String implements fmt.Stringer for a VersionEdit.
+func (v *VersionEdit) String() string {
+	return v.string(false /* verbose */, base.DefaultFormatter)
+}
+
 // Encode encodes an edit to the specified writer.
 func (v *VersionEdit) Encode(w io.Writer) error {
 	e := versionEditEncoder{new(bytes.Buffer)}
@@ -429,6 +526,15 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 		e.writeUvarint(tagNextFileNumber)
 		e.writeUvarint(uint64(v.NextFileNum))
 	}
+	for _, dfn := range v.RemovedBackingTables {
+		e.writeUvarint(tagRemovedBackingTable)
+		e.writeUvarint(uint64(dfn.FileNum()))
+	}
+	for _, fileBacking := range v.CreatedBackingTables {
+		e.writeUvarint(tagCreatedBackingTable)
+		e.writeUvarint(uint64(fileBacking.DiskFileNum.FileNum()))
+		e.writeUvarint(fileBacking.Size)
+	}
 	// RocksDB requires LastSeqNum to be encoded for the first MANIFEST entry,
 	// even though its value is zero. We detect this by encoding LastSeqNum when
 	// ComparerName is set.
@@ -442,7 +548,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 		e.writeUvarint(uint64(x.FileNum))
 	}
 	for _, x := range v.NewFiles {
-		customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0
+		customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0 || x.Meta.Virtual
 		var tag uint64
 		switch {
 		case x.Meta.HasRangeKeys:
@@ -495,6 +601,10 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 				e.writeUvarint(customTagNeedsCompaction)
 				e.writeBytes([]byte{1})
 			}
+			if x.Meta.Virtual {
+				e.writeUvarint(customTagVirtual)
+				e.writeUvarint(uint64(x.Meta.FileBacking.DiskFileNum.FileNum()))
+			}
 			e.writeUvarint(customTagTerminate)
 		}
 	}
@@ -502,6 +612,7 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 	return err
 }
 
+// versionEditDecoder should be used to decode version edits.
 type versionEditDecoder struct {
 	byteReader
 }
@@ -581,10 +692,30 @@ func (e versionEditEncoder) writeUvarint(u uint64) {
 
 // BulkVersionEdit summarizes the files added and deleted from a set of version
 // edits.
+//
+// INVARIANTS:
+// No file can be added to a level more than once. This is true globally, and
+// also true for all of the calls to Accumulate for a single bulk version edit.
+//
+// No file can be removed from a level more than once. This is true globally,
+// and also true for all of the calls to Accumulate for a single bulk version
+// edit.
+//
+// A file must not be added and removed from a given level in the same version
+// edit.
+//
+// A file that is being removed from a level must have been added to that level
+// before (in a prior version edit). Note that a given file can be deleted from
+// a level and added to another level in a single version edit
 type BulkVersionEdit struct {
-	Added   [NumLevels][]*FileMetadata
+	Added   [NumLevels]map[base.FileNum]*FileMetadata
 	Deleted [NumLevels]map[base.FileNum]*FileMetadata
 
+	// AddedFileBacking is a map to support lookup so that we can populate the
+	// FileBacking of virtual sstables during manifest replay.
+	AddedFileBacking   map[base.DiskFileNum]*FileBacking
+	RemovedFileBacking []base.DiskFileNum
+
 	// AddedByFileNum maps file number to file metadata for all added files
 	// from accumulated version edits. AddedByFileNum is only populated if set
 	// to non-nil by a caller. It must be set to non-nil when replaying
@@ -603,6 +734,19 @@ type BulkVersionEdit struct {
 
 // Accumulate adds the file addition and deletions in the specified version
 // edit to the bulk edit's internal state.
+//
+// INVARIANTS:
+// If a file is added to a given level in a call to Accumulate and then removed
+// from that level in a subsequent call, the file will not be present in the
+// resulting BulkVersionEdit.Deleted for that level.
+//
+// After accumulation of version edits, the bulk version edit may have
+// information about a file which has been deleted from a level, but it may
+// not have information about the same file added to the same level. The add
+// could've occurred as part of a previous bulk version edit. In this case,
+// the deleted file must be present in BulkVersionEdit.Deleted, at the end
+// of the accumulation, because we need to decrease the refcount of the
+// deleted file in Apply.
 func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 	for df, m := range ve.DeletedFiles {
 		dmap := b.Deleted[df.Level]
@@ -624,7 +768,28 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 		if m.MarkedForCompaction {
 			b.MarkedForCompactionCountDiff--
 		}
-		dmap[df.FileNum] = m
+		if _, ok := b.Added[df.Level][df.FileNum]; !ok {
+			dmap[df.FileNum] = m
+		} else {
+			// Present in b.Added for the same level.
+			delete(b.Added[df.Level], df.FileNum)
+		}
+	}
+
+	// Generate state for Added backing files. Note that these must be generated
+	// before we loop through the NewFiles, because we need to populate the
+	// FileBackings which might be used by the NewFiles loop.
+	if b.AddedFileBacking == nil {
+		b.AddedFileBacking = make(map[base.DiskFileNum]*FileBacking)
+	}
+	for _, fb := range ve.CreatedBackingTables {
+		if _, ok := b.AddedFileBacking[fb.DiskFileNum]; ok {
+			// There is already a FileBacking associated with fb.DiskFileNum.
+			// This should never happen. There must always be only one FileBacking
+			// associated with a backing sstable.
+			panic(fmt.Sprintf("pebble: duplicate file backing %s", fb.DiskFileNum.String()))
+		}
+		b.AddedFileBacking[fb.DiskFileNum] = fb
 	}
 
 	for _, nf := range ve.NewFiles {
@@ -635,7 +800,21 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 				return base.CorruptionErrorf("pebble: file deleted L%d.%s before it was inserted", nf.Level, nf.Meta.FileNum)
 			}
 		}
-		b.Added[nf.Level] = append(b.Added[nf.Level], nf.Meta)
+		if nf.Meta.Virtual && nf.Meta.FileBacking == nil {
+			// FileBacking for a virtual sstable must only be nil if we're performing
+			// manifest replay.
+			nf.Meta.FileBacking = b.AddedFileBacking[nf.BackingFileNum]
+			if nf.Meta.FileBacking == nil {
+				return errors.Errorf("FileBacking for virtual sstable must not be nil")
+			}
+		} else if nf.Meta.FileBacking == nil {
+			return errors.Errorf("Added file L%d.%s's has no FileBacking", nf.Level, nf.Meta.FileNum)
+		}
+
+		if b.Added[nf.Level] == nil {
+			b.Added[nf.Level] = make(map[base.FileNum]*FileMetadata)
+		}
+		b.Added[nf.Level][nf.Meta.FileNum] = nf.Meta
 		if b.AddedByFileNum != nil {
 			b.AddedByFileNum[nf.Meta.FileNum] = nf.Meta
 		}
@@ -643,36 +822,101 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 			b.MarkedForCompactionCountDiff++
 		}
 	}
+
+	// Since a file can be removed from backing files in exactly one version
+	// edit it is safe to just append without any de-duplication.
+	b.RemovedFileBacking = append(b.RemovedFileBacking, ve.RemovedBackingTables...)
+
 	return nil
 }
 
+// AccumulateIncompleteAndApplySingleVE should be called if a single version edit
+// is to be applied to the provided curr Version and if the caller needs to
+// update the versionSet.zombieTables map. This function exists separately from
+// BulkVersionEdit.Apply because it is easier to reason about properties
+// regarding BulkVersionedit.Accumulate/Apply and zombie table generation, if we
+// know that exactly one version edit is being accumulated.
+//
+// Note that the version edit passed into this function may be incomplete
+// because compactions don't have the ref counting information necessary to
+// populate VersionEdit.RemovedBackingTables. This function will complete such a
+// version edit by populating RemovedBackingTables.
+//
+// Invariant: Any file being deleted through ve must belong to the curr Version.
+// We can't have a delete for some arbitrary file which does not exist in curr.
+func AccumulateIncompleteAndApplySingleVE(
+	ve *VersionEdit,
+	curr *Version,
+	cmp Compare,
+	formatKey base.FormatKey,
+	flushSplitBytes int64,
+	readCompactionRate int64,
+	backingStateMap map[base.DiskFileNum]*FileBacking,
+	addBackingFunc func(*FileBacking),
+	removeBackingFunc func(base.DiskFileNum),
+	orderingInvariants OrderingInvariants,
+) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) {
+	if len(ve.RemovedBackingTables) != 0 {
+		panic("pebble: invalid incomplete version edit")
+	}
+	var b BulkVersionEdit
+	err := b.Accumulate(ve)
+	if err != nil {
+		return nil, nil, err
+	}
+	zombies = make(map[base.DiskFileNum]uint64)
+	v, err := b.Apply(
+		curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies, orderingInvariants,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for _, s := range b.AddedFileBacking {
+		addBackingFunc(s)
+	}
+
+	for fileNum := range zombies {
+		if _, ok := backingStateMap[fileNum]; ok {
+			// This table was backing some virtual sstable in the latest version,
+			// but is now a zombie. We add RemovedBackingTables entries for
+			// these, before the version edit is written to disk.
+			ve.RemovedBackingTables = append(
+				ve.RemovedBackingTables, fileNum,
+			)
+			removeBackingFunc(fileNum)
+		}
+	}
+	return v, zombies, nil
+}
+
 // Apply applies the delta b to the current version to produce a new
 // version. The new version is consistent with respect to the comparer cmp.
 //
 // curr may be nil, which is equivalent to a pointer to a zero version.
 //
-// On success, a map of zombie files containing the file numbers and sizes of
+// On success, if a non-nil zombies map is provided to Apply, the map is updated
-// deleted files is returned. These files are considered zombies because they
+// with file numbers and files sizes of deleted files. These files are
-// are no longer referenced by the returned Version, but cannot be deleted from
+// considered zombies because they are no longer referenced by the returned
-// disk as they are still in use by the incoming Version.
+// Version, but cannot be deleted from disk as they are still in use by the
+// incoming Version.
 func (b *BulkVersionEdit) Apply(
 	curr *Version,
 	cmp Compare,
 	formatKey base.FormatKey,
 	flushSplitBytes int64,
 	readCompactionRate int64,
-) (_ *Version, zombies map[base.FileNum]uint64, _ error) {
+	zombies map[base.DiskFileNum]uint64,
-	addZombie := func(fileNum base.FileNum, size uint64) {
+	orderingInvariants OrderingInvariants,
-		if zombies == nil {
+) (*Version, error) {
-			zombies = make(map[base.FileNum]uint64)
+	addZombie := func(state *FileBacking) {
-		}
-		zombies[fileNum] = size
-	}
-	// The remove zombie function is used to handle tables that are moved from
-	// one level to another during a version edit (i.e. a "move" compaction).
-	removeZombie := func(fileNum base.FileNum) {
 		if zombies != nil {
-			delete(zombies, fileNum)
+			zombies[state.DiskFileNum] = state.Size
+		}
+	}
+	removeZombie := func(state *FileBacking) {
+		if zombies != nil {
+			delete(zombies, state.DiskFileNum)
 		}
 	}
 
@@ -684,7 +928,7 @@ func (b *BulkVersionEdit) Apply(
 	}
 	v.Stats.MarkedForCompaction += b.MarkedForCompactionCountDiff
 	if v.Stats.MarkedForCompaction < 0 {
-		return nil, nil, base.CorruptionErrorf("pebble: version marked for compaction count negative")
+		return nil, base.CorruptionErrorf("pebble: version marked for compaction count negative")
 	}
 
 	for level := range v.Levels {
@@ -705,7 +949,7 @@ func (b *BulkVersionEdit) Apply(
 				// Initialize L0Sublevels.
 				if curr == nil || curr.L0Sublevels == nil {
 					if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
-						return nil, nil, errors.Wrap(err, "pebble: internal error")
+						return nil, errors.Wrap(err, "pebble: internal error")
 					}
 				} else {
 					v.L0Sublevels = curr.L0Sublevels
@@ -718,47 +962,70 @@ func (b *BulkVersionEdit) Apply(
 		// Some edits on this level.
 		lm := &v.Levels[level]
 		lmRange := &v.RangeKeyLevels[level]
-		addedFiles := b.Added[level]
+
-		deletedMap := b.Deleted[level]
+		addedFilesMap := b.Added[level]
-		if n := v.Levels[level].Len() + len(addedFiles); n == 0 {
+		deletedFilesMap := b.Deleted[level]
-			return nil, nil, base.CorruptionErrorf(
+		if n := v.Levels[level].Len() + len(addedFilesMap); n == 0 {
+			return nil, base.CorruptionErrorf(
 				"pebble: internal error: No current or added files but have deleted files: %d",
-				errors.Safe(len(deletedMap)))
+				errors.Safe(len(deletedFilesMap)))
 		}
 
-		// NB: addedFiles may be empty and it also is not necessarily
+		// NB: addedFilesMap may be empty. If a file is present in addedFilesMap
-		// internally consistent: it does not reflect deletions in deletedMap.
+		// for a level, it won't be present in deletedFilesMap for the same
+		// level.
 
-		for _, f := range deletedMap {
+		for _, f := range deletedFilesMap {
-			addZombie(f.FileNum, f.Size)
+			if obsolete := v.Levels[level].remove(f); obsolete {
-			if obsolete := v.Levels[level].tree.Delete(f); obsolete {
 				// Deleting a file from the B-Tree may decrement its
 				// reference count. However, because we cloned the
 				// previous level's B-Tree, this should never result in a
 				// file's reference count dropping to zero.
 				err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during B-Tree removal", level, f.FileNum)
-				return nil, nil, err
+				return nil, err
 			}
 			if f.HasRangeKeys {
-				if obsolete := v.RangeKeyLevels[level].tree.Delete(f); obsolete {
+				if obsolete := v.RangeKeyLevels[level].remove(f); obsolete {
 					// Deleting a file from the B-Tree may decrement its
 					// reference count. However, because we cloned the
 					// previous level's B-Tree, this should never result in a
 					// file's reference count dropping to zero.
 					err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during range-key B-Tree removal", level, f.FileNum)
-					return nil, nil, err
+					return nil, err
 				}
 			}
+
+			// Note that a backing sst will only become a zombie if the
+			// references to it in the latest version is 0. We will remove the
+			// backing sst from the zombie list in the next loop if one of the
+			// addedFiles in any of the levels is referencing the backing sst.
+			// This is possible if a physical sstable is virtualized, or if it
+			// is moved.
+			latestRefCount := f.LatestRefs()
+			if latestRefCount <= 0 {
+				// If a file is present in deletedFilesMap for a level, then it
+				// must have already been added to the level previously, which
+				// means that its latest ref count cannot be 0.
+				err := errors.Errorf("pebble: internal error: incorrect latestRefs reference counting for file", f.FileNum)
+				return nil, err
+			} else if f.LatestUnref() == 0 {
+				addZombie(f.FileBacking)
+			}
 		}
 
+		addedFiles := make([]*FileMetadata, 0, len(addedFilesMap))
+		for _, f := range addedFilesMap {
+			addedFiles = append(addedFiles, f)
+		}
+		// Sort addedFiles by file number. This isn't necessary, but tests which
+		// replay invalid manifests check the error output, and the error output
+		// depends on the order in which files are added to the btree.
+		sort.Slice(addedFiles, func(i, j int) bool {
+			return addedFiles[i].FileNum < addedFiles[j].FileNum
+		})
+
 		var sm, la *FileMetadata
 		for _, f := range addedFiles {
-			if _, ok := deletedMap[f.FileNum]; ok {
-				// Already called addZombie on this file in the preceding
-				// loop, so we don't need to do it here.
-				continue
-			}
-
 			// NB: allowedSeeks is used for read triggered compactions. It is set using
 			// Options.Experimental.ReadCompactionRate which defaults to 32KB.
 			var allowedSeeks int64
@@ -768,20 +1035,23 @@ func (b *BulkVersionEdit) Apply(
 			if allowedSeeks < 100 {
 				allowedSeeks = 100
 			}
-			atomic.StoreInt64(&f.Atomic.AllowedSeeks, allowedSeeks)
+			f.AllowedSeeks.Store(allowedSeeks)
 			f.InitAllowedSeeks = allowedSeeks
 
-			err := lm.tree.Insert(f)
+			err := lm.insert(f)
+			// We're adding this file to the new version, so increment the
+			// latest refs count.
+			f.LatestRef()
 			if err != nil {
-				return nil, nil, errors.Wrap(err, "pebble")
+				return nil, errors.Wrap(err, "pebble")
 			}
 			if f.HasRangeKeys {
-				err = lmRange.tree.Insert(f)
+				err = lmRange.insert(f)
 				if err != nil {
-					return nil, nil, errors.Wrap(err, "pebble")
+					return nil, errors.Wrap(err, "pebble")
 				}
 			}
-			removeZombie(f.FileNum)
+			removeZombie(f.FileBacking)
 			// Track the keys with the smallest and largest keys, so that we can
 			// check consistency of the modified span.
 			if sm == nil || base.InternalCompare(cmp, sm.Smallest, f.Smallest) > 0 {
@@ -793,27 +1063,36 @@ func (b *BulkVersionEdit) Apply(
 		}
 
 		if level == 0 {
-			if curr != nil && curr.L0Sublevels != nil && len(deletedMap) == 0 {
+			if curr != nil && curr.L0Sublevels != nil && len(deletedFilesMap) == 0 {
 				// Flushes and ingestions that do not delete any L0 files do not require
 				// a regeneration of L0Sublevels from scratch. We can instead generate
 				// it incrementally.
 				var err error
 				// AddL0Files requires addedFiles to be sorted in seqnum order.
-				addedFiles = append([]*FileMetadata(nil), addedFiles...)
 				SortBySeqNum(addedFiles)
 				v.L0Sublevels, err = curr.L0Sublevels.AddL0Files(addedFiles, flushSplitBytes, &v.Levels[0])
 				if errors.Is(err, errInvalidL0SublevelsOpt) {
 					err = v.InitL0Sublevels(cmp, formatKey, flushSplitBytes)
+				} else if invariants.Enabled && err == nil {
+					copyOfSublevels, err := NewL0Sublevels(&v.Levels[0], cmp, formatKey, flushSplitBytes)
+					if err != nil {
+						panic(fmt.Sprintf("error when regenerating sublevels: %s", err))
+					}
+					s1 := describeSublevels(base.DefaultFormatter, false /* verbose */, copyOfSublevels.Levels)
+					s2 := describeSublevels(base.DefaultFormatter, false /* verbose */, v.L0Sublevels.Levels)
+					if s1 != s2 {
+						panic(fmt.Sprintf("incremental L0 sublevel generation produced different output than regeneration: %s != %s", s1, s2))
+					}
 				}
 				if err != nil {
-					return nil, nil, errors.Wrap(err, "pebble: internal error")
+					return nil, errors.Wrap(err, "pebble: internal error")
 				}
 				v.L0SublevelFiles = v.L0Sublevels.Levels
 			} else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
-				return nil, nil, errors.Wrap(err, "pebble: internal error")
+				return nil, errors.Wrap(err, "pebble: internal error")
 			}
-			if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil {
+			if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter(), orderingInvariants); err != nil {
-				return nil, nil, errors.Wrap(err, "pebble: internal error")
+				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 			continue
 		}
@@ -833,10 +1112,10 @@ func (b *BulkVersionEdit) Apply(
 					end.Prev()
 				}
 			})
-			if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil {
+			if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter(), orderingInvariants); err != nil {
-				return nil, nil, errors.Wrap(err, "pebble: internal error")
+				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 		}
 	}
-	return v, zombies, nil
+	return v, nil
 }

vendor/github.com/cockroachdb/pebble/internal/private/sstable.go

@@ -8,7 +8,7 @@ import "github.com/cockroachdb/pebble/internal/base"
 
 // SSTableCacheOpts is a hook for specifying cache options to
 // sstable.NewReader.
-var SSTableCacheOpts func(cacheID uint64, fileNum base.FileNum) interface{}
+var SSTableCacheOpts func(cacheID uint64, fileNum base.DiskFileNum) interface{}
 
 // SSTableRawTombstonesOpt is a sstable.Reader option for disabling
 // fragmentation of the range tombstones returned by

vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go

@@ -52,15 +52,15 @@ import (
 //	                        │
 //	                        ╰── <?>.Next
 type UserIteratorConfig struct {
-	snapshot   uint64
+	snapshot     uint64
-	comparer   *base.Comparer
+	comparer     *base.Comparer
-	miter      keyspan.MergingIter
+	miter        keyspan.MergingIter
-	biter      keyspan.BoundedIter
+	biter        keyspan.BoundedIter
-	diter      keyspan.DefragmentingIter
+	diter        keyspan.DefragmentingIter
-	liters     [manifest.NumLevels]keyspan.LevelIter
+	liters       [manifest.NumLevels]keyspan.LevelIter
-	litersUsed int
+	litersUsed   int
-	onlySets   bool
+	internalKeys bool
-	bufs       *Buffers
+	bufs         *Buffers
 }
 
 // Buffers holds various buffers used for range key iteration. They're exposed
@@ -79,9 +79,10 @@ func (bufs *Buffers) PrepareForReuse() {
 
 // Init initializes the range key iterator stack for user iteration. The
 // resulting fragment iterator applies range key semantics, defragments spans
-// according to their user-observable state and, if onlySets = true, removes all
+// according to their user-observable state and, if !internalKeys, removes all
 // Keys other than RangeKeySets describing the current state of range keys. The
-// resulting spans contain Keys sorted by Suffix.
+// resulting spans contain Keys sorted by suffix (unless internalKeys is true,
+// in which case they remain sorted by trailer descending).
 //
 // The snapshot sequence number parameter determines which keys are visible. Any
 // keys not visible at the provided snapshot are ignored.
@@ -91,16 +92,20 @@ func (ui *UserIteratorConfig) Init(
 	lower, upper []byte,
 	hasPrefix *bool,
 	prefix *[]byte,
-	onlySets bool,
+	internalKeys bool,
 	bufs *Buffers,
 	iters ...keyspan.FragmentIterator,
 ) keyspan.FragmentIterator {
 	ui.snapshot = snapshot
 	ui.comparer = comparer
-	ui.onlySets = onlySets
+	ui.internalKeys = internalKeys
 	ui.miter.Init(comparer.Compare, ui, &bufs.merging, iters...)
 	ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix)
-	ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	if internalKeys {
+		ui.diter.Init(comparer, &ui.biter, keyspan.DefragmentInternal, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	} else {
+		ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	}
 	ui.litersUsed = 0
 	ui.bufs = bufs
 	return &ui.diter
@@ -136,7 +141,8 @@ func (ui *UserIteratorConfig) SetBounds(lower, upper []byte) {
 // of unset keys, removal of keys overwritten by a set at the same suffix, etc)
 // and then non-RangeKeySet keys are removed. The resulting transformed spans
 // only contain RangeKeySets describing the state visible at the provided
-// sequence number, and hold their Keys sorted by Suffix.
+// sequence number, and hold their Keys sorted by Suffix (except if internalKeys
+// is true, then keys remain sorted by trailer.
 func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error {
 	// Apply shadowing of keys.
 	dst.Start = s.Start
@@ -148,9 +154,16 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 	if err := coalesce(ui.comparer.Equal, &ui.bufs.sortBuf, ui.snapshot, s.Keys); err != nil {
 		return err
 	}
-	// During user iteration over range keys, unsets and deletes don't matter.
+	if ui.internalKeys {
-	// Remove them if onlySets = true. This step helps logical defragmentation
+		if s.KeysOrder != keyspan.ByTrailerDesc {
-	// during iteration.
+			panic("unexpected key ordering in UserIteratorTransform with internalKeys = true")
+		}
+		dst.Keys = ui.bufs.sortBuf.Keys
+		keyspan.SortKeysByTrailer(&dst.Keys)
+		return nil
+	}
+	// During user iteration over range keys, unsets and deletes don't matter. This
+	// step helps logical defragmentation during iteration.
 	keys := ui.bufs.sortBuf.Keys
 	dst.Keys = dst.Keys[:0]
 	for i := range keys {
@@ -164,17 +177,11 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
 				panic("pebble: keys unexpectedly not in ascending suffix order")
 			}
-			if ui.onlySets {
+			// Skip.
-				// Skip.
+			continue
-				continue
-			}
-			dst.Keys = append(dst.Keys, keys[i])
 		case base.InternalKeyKindRangeKeyDelete:
-			if ui.onlySets {
+			// Skip.
-				// Skip.
+			continue
-				continue
-			}
-			dst.Keys = append(dst.Keys, keys[i])
 		default:
 			return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
 		}
@@ -193,6 +200,10 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 // sequence numbers). It's intended for use during user iteration, when the
 // wrapped keyspan iterator is merging spans across all levels of the LSM.
 func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.Span) bool {
+	// This method is not called with internalKeys = true.
+	if ui.internalKeys {
+		panic("unexpected call to ShouldDefragment with internalKeys = true")
+	}
 	// This implementation must only be used on spans that have transformed by
 	// ui.Transform. The transform applies shadowing, removes all keys besides
 	// the resulting Sets and sorts the keys by suffix. Since shadowing has been
@@ -208,8 +219,8 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
 	ret := true
 	for i := range a.Keys {
 		if invariants.Enabled {
-			if ui.onlySets && (a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
+			if a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
-				b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet) {
+				b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet {
 				panic("pebble: unexpected non-RangeKeySet during defragmentation")
 			}
 			if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 ||

vendor/github.com/cockroachdb/pebble/internal/rate/rate.go

@@ -1,387 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package rate provides a rate limiter.
-package rate // import "github.com/cockroachdb/pebble/internal/rate"
-
-import (
-	"context"
-	"math"
-	"sync"
-	"time"
-
-	"github.com/cockroachdb/errors"
-)
-
-// Limit defines the maximum frequency of some events.
-// Limit is represented as number of events per second.
-// A zero Limit allows no events.
-type Limit float64
-
-// Inf is the infinite rate limit; it allows all events (even if burst is zero).
-const Inf = Limit(math.MaxFloat64)
-
-// Every converts a minimum time interval between events to a Limit.
-func Every(interval time.Duration) Limit {
-	if interval <= 0 {
-		return Inf
-	}
-	return 1 / Limit(interval.Seconds())
-}
-
-// A Limiter controls how frequently events are allowed to happen.
-// It implements a "token bucket" of size b, initially full and refilled
-// at rate r tokens per second.
-// Informally, in any large enough time interval, the Limiter limits the
-// rate to r tokens per second, with a maximum burst size of b events.
-// As a special case, if r == Inf (the infinite rate), b is ignored.
-// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets.
-//
-// The zero value is a valid Limiter, but it will reject all events.
-// Use NewLimiter to create non-zero Limiters.
-//
-// Limiter has three main methods, Allow, Reserve, and Wait.
-// Most callers should use Wait.
-//
-// Each of the three methods consumes a single token.
-// They differ in their behavior when no token is available.
-// If no token is available, Allow returns false.
-// If no token is available, Reserve returns a reservation for a future token
-// and the amount of time the caller must wait before using it.
-// If no token is available, Wait blocks until one can be obtained
-// or its associated context.Context is canceled.
-//
-// The methods AllowN, ReserveN, and WaitN consume n tokens.
-type Limiter struct {
-	limit Limit
-	burst int
-
-	mu     sync.Mutex
-	tokens float64
-	// last is the last time the limiter's tokens field was updated
-	last time.Time
-	// lastEvent is the latest time of a rate-limited event (past or future)
-	lastEvent time.Time
-}
-
-// Limit returns the maximum overall event rate.
-func (lim *Limiter) Limit() Limit {
-	lim.mu.Lock()
-	defer lim.mu.Unlock()
-	return lim.limit
-}
-
-// Burst returns the maximum burst size. Burst is the maximum number of tokens
-// that can be consumed in a single call to Allow, Reserve, or Wait, so higher
-// Burst values allow more events to happen at once.
-// A zero Burst allows no events, unless limit == Inf.
-func (lim *Limiter) Burst() int {
-	return lim.burst
-}
-
-// NewLimiter returns a new Limiter that allows events up to rate r and permits
-// bursts of at most b tokens.
-func NewLimiter(r Limit, b int) *Limiter {
-	return &Limiter{
-		limit: r,
-		burst: b,
-	}
-}
-
-// Allow is shorthand for AllowN(time.Now(), 1).
-func (lim *Limiter) Allow() bool {
-	return lim.AllowN(time.Now(), 1)
-}
-
-// AllowN reports whether n events may happen at time now.
-// Use this method if you intend to drop / skip events that exceed the rate limit.
-// Otherwise use Reserve or Wait.
-func (lim *Limiter) AllowN(now time.Time, n int) bool {
-	return lim.reserveN(now, n, 0).ok
-}
-
-// A Reservation holds information about events that are permitted by a Limiter to happen after a delay.
-// A Reservation may be canceled, which may enable the Limiter to permit additional events.
-type Reservation struct {
-	ok        bool
-	lim       *Limiter
-	tokens    int
-	timeToAct time.Time
-	// This is the Limit at reservation time, it can change later.
-	limit Limit
-}
-
-// OK returns whether the limiter can provide the requested number of tokens
-// within the maximum wait time.  If OK is false, Delay returns InfDuration, and
-// Cancel does nothing.
-func (r *Reservation) OK() bool {
-	return r.ok
-}
-
-// Delay is shorthand for DelayFrom(time.Now()).
-func (r *Reservation) Delay() time.Duration {
-	return r.DelayFrom(time.Now())
-}
-
-// InfDuration is the duration returned by Delay when a Reservation is not OK.
-const InfDuration = time.Duration(1<<63 - 1)
-
-// DelayFrom returns the duration for which the reservation holder must wait
-// before taking the reserved action.  Zero duration means act immediately.
-// InfDuration means the limiter cannot grant the tokens requested in this
-// Reservation within the maximum wait time.
-func (r *Reservation) DelayFrom(now time.Time) time.Duration {
-	if !r.ok {
-		return InfDuration
-	}
-	delay := r.timeToAct.Sub(now)
-	if delay < 0 {
-		return 0
-	}
-	return delay
-}
-
-// Cancel is shorthand for CancelAt(time.Now()).
-func (r *Reservation) Cancel() {
-	r.CancelAt(time.Now())
-}
-
-// CancelAt indicates that the reservation holder will not perform the reserved action
-// and reverses the effects of this Reservation on the rate limit as much as possible,
-// considering that other reservations may have already been made.
-func (r *Reservation) CancelAt(now time.Time) {
-	if !r.ok {
-		return
-	}
-
-	r.lim.mu.Lock()
-	defer r.lim.mu.Unlock()
-
-	if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(now) {
-		return
-	}
-
-	// calculate tokens to restore
-	// The duration between lim.lastEvent and r.timeToAct tells us how many tokens were reserved
-	// after r was obtained. These tokens should not be restored.
-	restoreTokens := float64(r.tokens) - r.limit.tokensFromDuration(r.lim.lastEvent.Sub(r.timeToAct))
-	if restoreTokens <= 0 {
-		return
-	}
-	// advance time to now
-	now, _, tokens := r.lim.advance(now)
-	// calculate new number of tokens
-	tokens += restoreTokens
-	if burst := float64(r.lim.burst); tokens > burst {
-		tokens = burst
-	}
-	// update state
-	r.lim.last = now
-	r.lim.tokens = tokens
-	if r.timeToAct == r.lim.lastEvent {
-		prevEvent := r.timeToAct.Add(r.limit.durationFromTokens(float64(-r.tokens)))
-		if !prevEvent.Before(now) {
-			r.lim.lastEvent = prevEvent
-		}
-	}
-}
-
-// Reserve is shorthand for ReserveN(time.Now(), 1).
-func (lim *Limiter) Reserve() Reservation {
-	return lim.ReserveN(time.Now(), 1)
-}
-
-// ReserveN returns a Reservation that indicates how long the caller must wait before n events happen.
-// The Limiter takes this Reservation into account when allowing future events.
-// ReserveN returns false if n exceeds the Limiter's burst size.
-// Usage example:
-//
-//	r := lim.ReserveN(time.Now(), 1)
-//	if !r.OK() {
-//	  // Not allowed to act! Did you remember to set lim.burst to be > 0 ?
-//	  return
-//	}
-//	time.Sleep(r.Delay())
-//	Act()
-//
-// Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events.
-// If you need to respect a deadline or cancel the delay, use Wait instead.
-// To drop or skip events exceeding rate limit, use Allow instead.
-func (lim *Limiter) ReserveN(now time.Time, n int) Reservation {
-	return lim.reserveN(now, n, InfDuration)
-}
-
-// Wait is shorthand for WaitN(ctx, 1).
-func (lim *Limiter) Wait(ctx context.Context) (err error) {
-	return lim.WaitN(ctx, 1)
-}
-
-// WaitN blocks until lim permits n events to happen.
-// It returns an error if n exceeds the Limiter's burst size, the Context is
-// canceled, or the expected wait time exceeds the Context's Deadline.
-func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
-	if n > lim.burst && lim.limit != Inf {
-		return errors.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", errors.Safe(n), errors.Safe(lim.burst))
-	}
-	// Check if ctx is already cancelled
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	default:
-	}
-	// Determine wait limit
-	now := time.Now()
-	waitLimit := InfDuration
-	if deadline, ok := ctx.Deadline(); ok {
-		waitLimit = deadline.Sub(now)
-	}
-	// Reserve
-	r := lim.reserveN(now, n, waitLimit)
-	if !r.ok {
-		return errors.Errorf("rate: Wait(n=%d) would exceed context deadline", errors.Safe(n))
-	}
-	// Wait
-	d := r.DelayFrom(now)
-	if d <= 0 {
-		return nil
-	}
-	t := time.NewTimer(d)
-	defer t.Stop()
-	select {
-	case <-t.C:
-		// We can proceed.
-		return nil
-	case <-ctx.Done():
-		// Context was canceled before we could proceed.  Cancel the
-		// reservation, which may permit other events to proceed sooner.
-		r.Cancel()
-		return ctx.Err()
-	}
-}
-
-// Delay is shorthand for DelayN(time.Now(), 1).
-func (lim *Limiter) Delay() time.Duration {
-	return lim.DelayN(time.Now(), 1)
-}
-
-// DelayN returns the delay to wait to permit n events to happen. Zero duration
-// means act immediately. InfDuration means the limiter cannot grant the tokens
-// requested within the maximum wait time.
-func (lim *Limiter) DelayN(now time.Time, n int) time.Duration {
-	r := lim.reserveN(now, n, InfDuration)
-	return r.DelayFrom(now)
-}
-
-// SetLimit is shorthand for SetLimitAt(time.Now(), newLimit).
-func (lim *Limiter) SetLimit(newLimit Limit) {
-	lim.SetLimitAt(time.Now(), newLimit)
-}
-
-// SetLimitAt sets a new Limit for the limiter. The new Limit, and Burst, may be violated
-// or underutilized by those which reserved (using Reserve or Wait) but did not yet act
-// before SetLimitAt was called.
-func (lim *Limiter) SetLimitAt(now time.Time, newLimit Limit) {
-	lim.mu.Lock()
-	defer lim.mu.Unlock()
-
-	now, _, tokens := lim.advance(now)
-
-	lim.last = now
-	lim.tokens = tokens
-	lim.limit = newLimit
-}
-
-// reserveN is a helper method for AllowN, ReserveN, and WaitN.
-// maxFutureReserve specifies the maximum reservation wait duration allowed.
-// reserveN returns Reservation, not *Reservation, to avoid allocation in AllowN and WaitN.
-func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duration) Reservation {
-	lim.mu.Lock()
-
-	if lim.limit == Inf {
-		lim.mu.Unlock()
-		return Reservation{
-			ok:        true,
-			lim:       lim,
-			tokens:    n,
-			timeToAct: now,
-		}
-	}
-
-	now, last, tokens := lim.advance(now)
-
-	// Calculate the remaining number of tokens resulting from the request.
-	tokens -= float64(n)
-
-	// Calculate the wait duration
-	var waitDuration time.Duration
-	if tokens < 0 {
-		waitDuration = lim.limit.durationFromTokens(-tokens)
-	}
-
-	// Decide result
-	ok := n <= lim.burst && waitDuration <= maxFutureReserve
-
-	// Prepare reservation
-	r := Reservation{
-		ok:    ok,
-		lim:   lim,
-		limit: lim.limit,
-	}
-	if ok {
-		r.tokens = n
-		r.timeToAct = now.Add(waitDuration)
-	}
-
-	// Update state
-	if ok {
-		lim.last = now
-		lim.tokens = tokens
-		lim.lastEvent = r.timeToAct
-	} else {
-		lim.last = last
-	}
-
-	lim.mu.Unlock()
-	return r
-}
-
-// advance calculates and returns an updated state for lim resulting from the passage of time.
-// lim is not changed.
-func (lim *Limiter) advance(
-	now time.Time,
-) (newNow time.Time, newLast time.Time, newTokens float64) {
-	last := lim.last
-	if now.Before(last) {
-		last = now
-	}
-
-	// Avoid making delta overflow below when last is very old.
-	maxElapsed := lim.limit.durationFromTokens(float64(lim.burst) - lim.tokens)
-	elapsed := now.Sub(last)
-	if elapsed > maxElapsed {
-		elapsed = maxElapsed
-	}
-
-	// Calculate the new number of tokens, due to time that passed.
-	delta := lim.limit.tokensFromDuration(elapsed)
-	tokens := lim.tokens + delta
-	if burst := float64(lim.burst); tokens > burst {
-		tokens = burst
-	}
-
-	return now, last, tokens
-}
-
-// durationFromTokens is a unit conversion function from the number of tokens to the duration
-// of time it takes to accumulate them at a rate of limit tokens per second.
-func (limit Limit) durationFromTokens(tokens float64) time.Duration {
-	seconds := tokens / float64(limit)
-	return time.Nanosecond * time.Duration(1e9*seconds)
-}
-
-// tokensFromDuration is a unit conversion function from a time duration to the number of tokens
-// which could be accumulated during that duration at a rate of limit tokens per second.
-func (limit Limit) tokensFromDuration(d time.Duration) float64 {
-	return d.Seconds() * float64(limit)
-}

vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go

@@ -20,17 +20,27 @@ import (
 	"strings"
 
 	"github.com/cockroachdb/pebble/internal/base"
+	"golang.org/x/exp/constraints"
+	"golang.org/x/exp/rand"
 )
 
 const alpha = "abcdefghijklmnopqrstuvwxyz"
 
 const suffixDelim = '@'
 
+var inverseAlphabet = make(map[byte]int64, len(alpha))
+
+func init() {
+	for i := range alpha {
+		inverseAlphabet[alpha[i]] = int64(i)
+	}
+}
+
 // MaxSuffixLen is the maximum length of a suffix generated by this package.
-var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", math.MaxInt64))
+var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", int64(math.MaxInt64)))
 
 // Comparer is the comparer for test keys generated by this package.
-var Comparer *base.Comparer = &base.Comparer{
+var Comparer = &base.Comparer{
 	Compare: compare,
 	Equal:   func(a, b []byte) bool { return compare(a, b) == 0 },
 	AbbreviatedKey: func(k []byte) uint64 {
@@ -136,7 +146,7 @@ func compareTimestamps(a, b []byte) int {
 // Keyspace describes a finite keyspace of unsuffixed test keys.
 type Keyspace interface {
 	// Count returns the number of keys that exist within this keyspace.
-	Count() int
+	Count() int64
 
 	// MaxLen returns the maximum length, in bytes, of a key within this
 	// keyspace. This is only guaranteed to return an upper bound.
@@ -144,20 +154,21 @@ type Keyspace interface {
 
 	// Slice returns the sub-keyspace from index i, inclusive, to index j,
 	// exclusive. The receiver is unmodified.
-	Slice(i, j int) Keyspace
+	Slice(i, j int64) Keyspace
 
 	// EveryN returns a key space that includes 1 key for every N keys in the
 	// original keyspace. The receiver is unmodified.
-	EveryN(n int) Keyspace
+	EveryN(n int64) Keyspace
 
-	key(buf []byte, i int) int
+	// key writes the i-th key to the buffer and returns the length.
+	key(buf []byte, i int64) int
 }
 
 // Divvy divides the provided keyspace into N equal portions, containing
 // disjoint keys evenly distributed across the keyspace.
-func Divvy(ks Keyspace, n int) []Keyspace {
+func Divvy(ks Keyspace, n int64) []Keyspace {
 	ret := make([]Keyspace, n)
-	for i := 0; i < n; i++ {
+	for i := int64(0); i < n; i++ {
 		ret[i] = ks.Slice(i, ks.Count()).EveryN(n)
 	}
 	return ret
@@ -175,7 +186,7 @@ func Alpha(maxLength int) Keyspace {
 
 // KeyAt returns the i-th key within the keyspace with a suffix encoding the
 // timestamp t.
-func KeyAt(k Keyspace, i int, t int) []byte {
+func KeyAt(k Keyspace, i int64, t int64) []byte {
 	b := make([]byte, k.MaxLen()+MaxSuffixLen)
 	return b[:WriteKeyAt(b, k, i, t)]
 }
@@ -183,20 +194,20 @@ func KeyAt(k Keyspace, i int, t int) []byte {
 // WriteKeyAt writes the i-th key within the keyspace to the buffer dst, with a
 // suffix encoding the timestamp t suffix. It returns the number of bytes
 // written.
-func WriteKeyAt(dst []byte, k Keyspace, i int, t int) int {
+func WriteKeyAt(dst []byte, k Keyspace, i int64, t int64) int {
 	n := WriteKey(dst, k, i)
 	n += WriteSuffix(dst[n:], t)
 	return n
 }
 
 // Suffix returns the test keys suffix representation of timestamp t.
-func Suffix(t int) []byte {
+func Suffix(t int64) []byte {
 	b := make([]byte, MaxSuffixLen)
 	return b[:WriteSuffix(b, t)]
 }
 
 // SuffixLen returns the exact length of the given suffix when encoded.
-func SuffixLen(t int) int {
+func SuffixLen(t int64) int {
 	// Begin at 1 for the '@' delimiter, 1 for a single digit.
 	n := 2
 	t /= 10
@@ -208,42 +219,42 @@ func SuffixLen(t int) int {
 }
 
 // ParseSuffix returns the integer representation of the encoded suffix.
-func ParseSuffix(s []byte) (int, error) {
+func ParseSuffix(s []byte) (int64, error) {
-	return strconv.Atoi(strings.TrimPrefix(string(s), string(suffixDelim)))
+	return strconv.ParseInt(strings.TrimPrefix(string(s), string(suffixDelim)), 10, 64)
 }
 
 // WriteSuffix writes the test keys suffix representation of timestamp t to dst,
 // returning the number of bytes written.
-func WriteSuffix(dst []byte, t int) int {
+func WriteSuffix(dst []byte, t int64) int {
 	dst[0] = suffixDelim
 	n := 1
-	n += len(strconv.AppendInt(dst[n:n], int64(t), 10))
+	n += len(strconv.AppendInt(dst[n:n], t, 10))
 	return n
 }
 
 // Key returns the i-th unsuffixed key within the keyspace.
-func Key(k Keyspace, i int) []byte {
+func Key(k Keyspace, i int64) []byte {
 	b := make([]byte, k.MaxLen())
 	return b[:k.key(b, i)]
 }
 
 // WriteKey writes the i-th unsuffixed key within the keyspace to the buffer dst. It
 // returns the number of bytes written.
-func WriteKey(dst []byte, k Keyspace, i int) int {
+func WriteKey(dst []byte, k Keyspace, i int64) int {
 	return k.key(dst, i)
 }
 
 type alphabet struct {
 	alphabet  []byte
 	maxLength int
-	headSkip  int
+	headSkip  int64
-	tailSkip  int
+	tailSkip  int64
-	increment int
+	increment int64
 }
 
-func (a alphabet) Count() int {
+func (a alphabet) Count() int64 {
 	// Calculate the total number of keys, ignoring the increment.
-	total := (keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip)
+	total := keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip
 
 	// The increment dictates that we take every N keys, where N = a.increment.
 	// Consider a total containing the 5 keys:
@@ -263,20 +274,25 @@ func (a alphabet) MaxLen() int {
 	return a.maxLength
 }
 
-func (a alphabet) Slice(i, j int) Keyspace {
+func (a alphabet) Slice(i, j int64) Keyspace {
 	s := a
 	s.headSkip += i
 	s.tailSkip += a.Count() - j
 	return s
 }
 
-func (a alphabet) EveryN(n int) Keyspace {
+func (a alphabet) EveryN(n int64) Keyspace {
 	s := a
 	s.increment *= n
 	return s
 }
 
-func keyCount(n, l int) int {
+func keyCount(n, l int) int64 {
+	if n == 0 {
+		return 0
+	} else if n == 1 {
+		return int64(l)
+	}
 	// The number of representable keys in the keyspace is a function of the
 	// length of the alphabet n and the max key length l. Consider how the
 	// number of representable keys grows as l increases:
@@ -286,10 +302,10 @@ func keyCount(n, l int) int {
 	// l = 3: n + n^2 + n^3
 	// ...
 	// Σ i=(1...l) n^i = n*(n^l - 1)/(n-1)
-	return (n * (int(math.Pow(float64(n), float64(l))) - 1)) / (n - 1)
+	return (int64(n) * (int64(math.Pow(float64(n), float64(l))) - 1)) / int64(n-1)
 }
 
-func (a alphabet) key(buf []byte, idx int) int {
+func (a alphabet) key(buf []byte, idx int64) int {
 	// This function generates keys of length 1..maxKeyLength, pulling
 	// characters from the alphabet. The idx determines which key to generate,
 	// generating the i-th lexicographically next key.
@@ -308,14 +324,14 @@ func (a alphabet) key(buf []byte, idx int) int {
 		keyCount(len(a.alphabet), a.maxLength))
 }
 
-func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int {
+func generateAlphabetKey(buf, alphabet []byte, i, keyCount int64) int {
 	if keyCount == 0 || i > keyCount || i < 0 {
 		return 0
 	}
 
 	// Of the keyCount keys in the generative keyspace, how many are there
 	// starting with a particular character?
-	keysPerCharacter := keyCount / len(alphabet)
+	keysPerCharacter := keyCount / int64(len(alphabet))
 
 	// Find the character that the key at index i starts with and set it.
 	characterIdx := i / keysPerCharacter
@@ -347,3 +363,156 @@ func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int {
 	i = i - keysPerCharacter*characterIdx - 1
 	return 1 + generateAlphabetKey(buf[1:], alphabet, i, keysPerCharacter-1)
 }
+
+// computeAlphabetKeyIndex computes the inverse of generateAlphabetKey,
+// returning the index of a particular key, given the provided alphabet and max
+// length of a key.
+//
+// len(key) must be ≥ 1.
+func computeAlphabetKeyIndex(key []byte, alphabet map[byte]int64, n int) int64 {
+	i, ok := alphabet[key[0]]
+	if !ok {
+		panic(fmt.Sprintf("unrecognized alphabet character %v", key[0]))
+	}
+	// How many keys exist that start with the preceding i characters? Each of
+	// the i characters themselves are a key, plus the count of all the keys
+	// with one less character for each.
+	ret := i + i*keyCount(len(alphabet), n-1)
+	if len(key) > 1 {
+		ret += 1 + computeAlphabetKeyIndex(key[1:], alphabet, n-1)
+	}
+	return ret
+}
+
+func abs(a int64) int64 {
+	if a < 0 {
+		return -a
+	}
+	return a
+}
+
+// RandomSeparator returns a random alphabetic key k such that a < k < b,
+// pulling randomness from the provided random number generator. If dst is
+// provided and the generated key fits within dst's capacity, the returned slice
+// will use dst's memory.
+//
+// If a prefix P exists such that Prefix(a) < P < Prefix(b), the generated key
+// will consist of the prefix P appended with the provided suffix. A zero suffix
+// generates an unsuffixed key. If no such prefix P exists, RandomSeparator will
+// try to find a key k with either Prefix(a) or Prefix(b) such that a < k < b,
+// but the generated key will not use the provided suffix. Note that it's
+// possible that no separator key exists (eg, a='a@2', b='a@1'), in which case
+// RandomSeparator returns nil.
+//
+// If RandomSeparator generates a new prefix, the generated prefix will have
+// length at most MAX(maxLength, len(Prefix(a)), len(Prefix(b))).
+//
+// RandomSeparator panics if a or b fails to decode.
+func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Rand) []byte {
+	if Comparer.Compare(a, b) >= 0 {
+		return nil
+	}
+
+	// Determine both keys' logical prefixes and suffixes.
+	ai := Comparer.Split(a)
+	bi := Comparer.Split(b)
+	ap := a[:ai]
+	bp := b[:bi]
+	maxLength = max[int](maxLength, max[int](len(ap), len(bp)))
+	var as, bs int64
+	var err error
+	if ai != len(a) {
+		as, err = ParseSuffix(a[ai:])
+		if err != nil {
+			panic(fmt.Sprintf("failed to parse suffix of %q", a))
+		}
+	}
+	if bi != len(b) {
+		bs, err = ParseSuffix(b[bi:])
+		if err != nil {
+			panic(fmt.Sprintf("failed to parse suffix of %q", b))
+		}
+	}
+
+	apIdx := computeAlphabetKeyIndex(ap, inverseAlphabet, maxLength)
+	bpIdx := computeAlphabetKeyIndex(bp, inverseAlphabet, maxLength)
+	diff := bpIdx - apIdx
+	generatedIdx := bpIdx
+	if diff > 0 {
+		var add int64 = diff + 1
+		var start int64 = apIdx
+		if as == 1 {
+			// There's no expressible key with prefix a greater than a@1. So,
+			// exclude ap.
+			start = apIdx + 1
+			add = diff
+		}
+		if bs == 0 {
+			// No key with prefix b can sort before b@0. We don't want to pick b.
+			add--
+		}
+		// We're allowing generated id to be in the range [start, start + add - 1].
+		if start > start+add-1 {
+			return nil
+		}
+		// If we can generate a key which is actually in the middle of apIdx
+		// and bpIdx use it so that we don't have to bother about timestamps.
+		generatedIdx = rng.Int63n(add) + start
+		for diff > 1 && generatedIdx == apIdx || generatedIdx == bpIdx {
+			generatedIdx = rng.Int63n(add) + start
+		}
+	}
+
+	switch {
+	case generatedIdx == apIdx && generatedIdx == bpIdx:
+		if abs(bs-as) <= 1 {
+			// There's no expressible suffix between the two, and there's no
+			// possible separator key.
+			return nil
+		}
+		// The key b is >= key a, but has the same prefix, so b must have the
+		// smaller timestamp, unless a has timestamp of 0.
+		//
+		// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
+		// any suffix we generate will be greater than it.
+		if as == 0 {
+			// bs > as
+			suffix = bs + rng.Int63n(10) + 1
+		} else {
+			// bs < as.
+			// Generate suffix in range [bs + 1, as - 1]
+			suffix = bs + 1 + rng.Int63n(as-bs-1)
+		}
+	case generatedIdx == apIdx:
+		// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
+		// any suffix we generate will be greater than it.
+		if as == 0 && suffix == 0 {
+			suffix++
+		} else if as != 0 && suffix >= as {
+			suffix = rng.Int63n(as)
+		}
+	case generatedIdx == bpIdx:
+		if suffix <= bs {
+			suffix = bs + rng.Int63n(10) + 1
+		}
+	}
+	if sz := maxLength + SuffixLen(suffix); cap(dst) < sz {
+		dst = make([]byte, sz)
+	} else {
+		dst = dst[:cap(dst)]
+	}
+	var w int
+	if suffix == 0 {
+		w = WriteKey(dst, Alpha(maxLength), generatedIdx)
+	} else {
+		w = WriteKeyAt(dst, Alpha(maxLength), generatedIdx, suffix)
+	}
+	return dst[:w]
+}
+
+func max[I constraints.Ordered](a, b I) I {
+	if b > a {
+		return b
+	}
+	return a
+}

vendor/github.com/cockroachdb/pebble/internal_iterator.go

@@ -1,332 +0,0 @@
-// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
-// of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package pebble
-
-import (
-	"context"
-
-	"github.com/cockroachdb/pebble/internal/base"
-	"github.com/cockroachdb/pebble/internal/keyspan"
-	"github.com/cockroachdb/pebble/internal/manifest"
-)
-
-// scanInternalIterator is an iterator that returns all internal keys instead of
-// collapsing them by user keys. For instance, an InternalKeyKindDelete would be
-// returned as an InternalKeyKindDelete instead of the iterator skipping over to
-// the next key. Useful if an external user of Pebble needs to observe and
-// rebuild Pebble's history of internal keys, such as in node-to-node
-// replication. For use with {db,snapshot}.ScanInternal().
-//
-// scanInternalIterator is allowed to ignorepoint keys deleted by range deletions,
-// and range keys shadowed by a range key unset or delete, however it *must*
-// return the range delete as well as the range key unset/delete that did the
-// shadowing.
-type scanInternalIterator struct {
-	opts            IterOptions
-	comparer        *base.Comparer
-	iter            internalIterator
-	readState       *readState
-	rangeKey        *iteratorRangeKeyState
-	pointKeyIter    keyspan.InterleavingIter
-	iterKey         *InternalKey
-	iterValue       LazyValue
-	alloc           *iterAlloc
-	newIters        tableNewIters
-	newIterRangeKey keyspan.TableNewSpanIter
-	seqNum          uint64
-
-	// boundsBuf holds two buffers used to store the lower and upper bounds.
-	// Whenever the InternalIterator's bounds change, the new bounds are copied
-	// into boundsBuf[boundsBufIdx]. The two bounds share a slice to reduce
-	// allocations. opts.LowerBound and opts.UpperBound point into this slice.
-	boundsBuf    [2][]byte
-	boundsBufIdx int
-}
-
-// TODO(sumeer): scanInternalImpl is the implementation for the user-facing
-// ScanInternal. Make it accept a context parameter, and plumb it through for
-// tracing support.
-
-func scanInternalImpl(
-	lower []byte,
-	iter *scanInternalIterator,
-	visitPointKey func(key *InternalKey, value LazyValue) error,
-	visitRangeDel func(start, end []byte, seqNum uint64) error,
-	visitRangeKey func(start, end []byte, keys []keyspan.Key) error,
-) error {
-	for valid := iter.seekGE(lower); valid && iter.error() == nil; valid = iter.next() {
-		key := iter.unsafeKey()
-
-		switch key.Kind() {
-		case InternalKeyKindRangeKeyDelete, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet:
-			span := iter.unsafeSpan()
-			if err := visitRangeKey(span.Start, span.End, span.Keys); err != nil {
-				return err
-			}
-		case InternalKeyKindRangeDelete:
-			rangeDel := iter.unsafeRangeDel()
-			if err := visitRangeDel(rangeDel.Start, rangeDel.End, rangeDel.LargestSeqNum()); err != nil {
-				return err
-			}
-		default:
-			val := iter.lazyValue()
-			if s := iter.pointKeyIter.Span(); s != nil && s.CoversAt(iter.seqNum, key.SeqNum()) {
-				// Key deleted by a range deletion. Skip it.
-				continue
-			}
-			if err := visitPointKey(key, val); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// constructPointIter constructs a merging iterator and sets i.iter to it.
-func (i *scanInternalIterator) constructPointIter(memtables flushableList, buf *iterAlloc) {
-	// Merging levels and levels from iterAlloc.
-	mlevels := buf.mlevels[:0]
-	levels := buf.levels[:0]
-
-	// We compute the number of levels needed ahead of time and reallocate a slice if
-	// the array from the iterAlloc isn't large enough. Doing this allocation once
-	// should improve the performance.
-	numMergingLevels := len(memtables)
-	numLevelIters := 0
-
-	current := i.readState.current
-	numMergingLevels += len(current.L0SublevelFiles)
-	numLevelIters += len(current.L0SublevelFiles)
-	for level := 1; level < len(current.Levels); level++ {
-		if current.Levels[level].Empty() {
-			continue
-		}
-		numMergingLevels++
-		numLevelIters++
-	}
-
-	if numMergingLevels > cap(mlevels) {
-		mlevels = make([]mergingIterLevel, 0, numMergingLevels)
-	}
-	if numLevelIters > cap(levels) {
-		levels = make([]levelIter, 0, numLevelIters)
-	}
-	// TODO(bilal): Push these into the iterAlloc buf.
-	var rangeDelMiter keyspan.MergingIter
-	rangeDelIters := make([]keyspan.FragmentIterator, 0, numMergingLevels)
-	rangeDelLevels := make([]keyspan.LevelIter, 0, numLevelIters)
-
-	// Next are the memtables.
-	for j := len(memtables) - 1; j >= 0; j-- {
-		mem := memtables[j]
-		mlevels = append(mlevels, mergingIterLevel{
-			iter: mem.newIter(&i.opts),
-		})
-		if rdi := mem.newRangeDelIter(&i.opts); rdi != nil {
-			rangeDelIters = append(rangeDelIters, rdi)
-		}
-	}
-
-	// Next are the file levels: L0 sub-levels followed by lower levels.
-	mlevelsIndex := len(mlevels)
-	levelsIndex := len(levels)
-	mlevels = mlevels[:numMergingLevels]
-	levels = levels[:numLevelIters]
-	rangeDelLevels = rangeDelLevels[:numLevelIters]
-	addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Level) {
-		li := &levels[levelsIndex]
-		rli := &rangeDelLevels[levelsIndex]
-
-		li.init(
-			context.Background(), i.opts, i.comparer.Compare, i.comparer.Split, i.newIters, files, level,
-			internalIterOpts{})
-		li.initBoundaryContext(&mlevels[mlevelsIndex].levelIterBoundaryContext)
-		mlevels[mlevelsIndex].iter = li
-		rli.Init(keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters},
-			i.comparer.Compare, tableNewRangeDelIter(context.Background(), i.newIters), files, level,
-			manifest.KeyTypePoint)
-		rangeDelIters = append(rangeDelIters, rli)
-
-		levelsIndex++
-		mlevelsIndex++
-	}
-
-	// Add level iterators for the L0 sublevels, iterating from newest to
-	// oldest.
-	for i := len(current.L0SublevelFiles) - 1; i >= 0; i-- {
-		addLevelIterForFiles(current.L0SublevelFiles[i].Iter(), manifest.L0Sublevel(i))
-	}
-
-	// Add level iterators for the non-empty non-L0 levels.
-	for level := 1; level < numLevels; level++ {
-		if current.Levels[level].Empty() {
-			continue
-		}
-		addLevelIterForFiles(current.Levels[level].Iter(), manifest.Level(level))
-	}
-	buf.merging.init(&i.opts, &InternalIteratorStats{}, i.comparer.Compare, i.comparer.Split, mlevels...)
-	buf.merging.snapshot = i.seqNum
-	rangeDelMiter.Init(i.comparer.Compare, keyspan.VisibleTransform(i.seqNum), new(keyspan.MergingBuffers), rangeDelIters...)
-	i.pointKeyIter.Init(i.comparer, &buf.merging, &rangeDelMiter, nil /* mask */, i.opts.LowerBound, i.opts.UpperBound)
-	i.iter = &i.pointKeyIter
-}
-
-// constructRangeKeyIter constructs the range-key iterator stack, populating
-// i.rangeKey.rangeKeyIter with the resulting iterator. This is similar to
-// Iterator.constructRangeKeyIter, except it doesn't handle batches and ensures
-// iterConfig does *not* elide unsets/deletes.
-func (i *scanInternalIterator) constructRangeKeyIter() {
-	// We want the bounded iter from iterConfig, but not the collapsing of
-	// RangeKeyUnsets and RangeKeyDels.
-	i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init(
-		i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound,
-		nil /* hasPrefix */, nil /* prefix */, false, /* onlySets */
-		&i.rangeKey.rangeKeyBuffers.internal)
-
-	// Next are the flushables: memtables and large batches.
-	for j := len(i.readState.memtables) - 1; j >= 0; j-- {
-		mem := i.readState.memtables[j]
-		// We only need to read from memtables which contain sequence numbers older
-		// than seqNum.
-		if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum {
-			continue
-		}
-		if rki := mem.newRangeKeyIter(&i.opts); rki != nil {
-			i.rangeKey.iterConfig.AddLevel(rki)
-		}
-	}
-
-	current := i.readState.current
-	// Next are the file levels: L0 sub-levels followed by lower levels.
-	//
-	// Add file-specific iterators for L0 files containing range keys. This is less
-	// efficient than using levelIters for sublevels of L0 files containing
-	// range keys, but range keys are expected to be sparse anyway, reducing the
-	// cost benefit of maintaining a separate L0Sublevels instance for range key
-	// files and then using it here.
-	//
-	// NB: We iterate L0's files in reverse order. They're sorted by
-	// LargestSeqNum ascending, and we need to add them to the merging iterator
-	// in LargestSeqNum descending to preserve the merging iterator's invariants
-	// around Key Trailer order.
-	iter := current.RangeKeyLevels[0].Iter()
-	for f := iter.Last(); f != nil; f = iter.Prev() {
-		spanIterOpts := &keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}
-		spanIter, err := i.newIterRangeKey(f, spanIterOpts)
-		if err != nil {
-			i.rangeKey.iterConfig.AddLevel(&errorKeyspanIter{err: err})
-			continue
-		}
-		i.rangeKey.iterConfig.AddLevel(spanIter)
-	}
-
-	// Add level iterators for the non-empty non-L0 levels.
-	for level := 1; level < len(current.RangeKeyLevels); level++ {
-		if current.RangeKeyLevels[level].Empty() {
-			continue
-		}
-		li := i.rangeKey.iterConfig.NewLevelIter()
-		spanIterOpts := keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}
-		li.Init(spanIterOpts, i.comparer.Compare, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(),
-			manifest.Level(level), manifest.KeyTypeRange)
-		i.rangeKey.iterConfig.AddLevel(li)
-	}
-}
-
-// seekGE seeks this iterator to the first key that's greater than or equal
-// to the specified user key.
-func (i *scanInternalIterator) seekGE(key []byte) bool {
-	i.iterKey, i.iterValue = i.iter.SeekGE(key, base.SeekGEFlagsNone)
-	return i.iterKey != nil
-}
-
-// unsafeKey returns the unsafe InternalKey at the current position. The value
-// is nil if the iterator is invalid or exhausted.
-func (i *scanInternalIterator) unsafeKey() *InternalKey {
-	return i.iterKey
-}
-
-// lazyValue returns a value pointer to the value at the current iterator
-// position. Behaviour undefined if unsafeKey() returns a Range key or Rangedel
-// kind key.
-func (i *scanInternalIterator) lazyValue() LazyValue {
-	return i.iterValue
-}
-
-// unsafeRangeDel returns a range key span. Behaviour undefined if UnsafeKey returns
-// a non-rangedel kind.
-func (i *scanInternalIterator) unsafeRangeDel() *keyspan.Span {
-	return i.pointKeyIter.Span()
-}
-
-// unsafeSpan returns a range key span. Behaviour undefined if UnsafeKey returns
-// a non-rangekey type.
-func (i *scanInternalIterator) unsafeSpan() *keyspan.Span {
-	return i.rangeKey.iiter.Span()
-}
-
-// next advances the iterator in the forward direction, and returns the
-// iterator's new validity state.
-func (i *scanInternalIterator) next() bool {
-	i.iterKey, i.iterValue = i.iter.Next()
-	return i.iterKey != nil
-}
-
-// error returns an error from the internal iterator, if there's any.
-func (i *scanInternalIterator) error() error {
-	return i.iter.Error()
-}
-
-// close closes this iterator, and releases any pooled objects.
-func (i *scanInternalIterator) close() error {
-	if err := i.iter.Close(); err != nil {
-		return err
-	}
-	i.readState.unref()
-	if i.rangeKey != nil {
-		i.rangeKey.PrepareForReuse()
-		*i.rangeKey = iteratorRangeKeyState{
-			rangeKeyBuffers: i.rangeKey.rangeKeyBuffers,
-		}
-		iterRangeKeyStateAllocPool.Put(i.rangeKey)
-		i.rangeKey = nil
-	}
-	if alloc := i.alloc; alloc != nil {
-		for j := range i.boundsBuf {
-			if cap(i.boundsBuf[j]) >= maxKeyBufCacheSize {
-				alloc.boundsBuf[j] = nil
-			} else {
-				alloc.boundsBuf[j] = i.boundsBuf[j]
-			}
-		}
-		*alloc = iterAlloc{
-			keyBuf:              alloc.keyBuf[:0],
-			boundsBuf:           alloc.boundsBuf,
-			prefixOrFullSeekKey: alloc.prefixOrFullSeekKey[:0],
-		}
-		iterAllocPool.Put(alloc)
-		i.alloc = nil
-	}
-	return nil
-}
-
-func (i *scanInternalIterator) initializeBoundBufs(lower, upper []byte) {
-	buf := i.boundsBuf[i.boundsBufIdx][:0]
-	if lower != nil {
-		buf = append(buf, lower...)
-		i.opts.LowerBound = buf
-	} else {
-		i.opts.LowerBound = nil
-	}
-	if upper != nil {
-		buf = append(buf, upper...)
-		i.opts.UpperBound = buf[len(buf)-len(upper):]
-	} else {
-		i.opts.UpperBound = nil
-	}
-	i.boundsBuf[i.boundsBufIdx] = buf
-	i.boundsBufIdx = 1 - i.boundsBufIdx
-}

vendor/github.com/cockroachdb/pebble/iterator.go

@@ -9,7 +9,6 @@ import (
 	"context"
 	"io"
 	"sync"
-	"sync/atomic"
 	"unsafe"
 
 	"github.com/cockroachdb/errors"
@@ -188,7 +187,9 @@ type Iterator struct {
 	comparer  base.Comparer
 	iter      internalIterator
 	pointIter internalIterator
+	// Either readState or version is set, but not both.
 	readState *readState
+	version   *version
 	// rangeKey holds iteration state specific to iteration over range keys.
 	// The range key field may be nil if the Iterator has never been configured
 	// to iterate over range keys. Its non-nilness cannot be used to determine
@@ -446,6 +447,11 @@ const (
 	seekPrefixGELastPositioningOp
 	seekGELastPositioningOp
 	seekLTLastPositioningOp
+	// internalNextOp is a special internal iterator positioning operation used
+	// by CanDeterministicallySingleDelete. It exists for enforcing requirements
+	// around calling CanDeterministicallySingleDelete at most once per external
+	// iterator position.
+	internalNextOp
 	// invalidatedLastPositionOp is similar to unknownLastPositionOp and the
 	// only reason to distinguish this is for the wider set of SeekGE
 	// optimizations we permit for the external iterator Iterator.forwardOnly
@@ -549,6 +555,21 @@ func (i *Iterator) findNextEntry(limit []byte) {
 			return
 		}
 
+		// If the user has configured a SkipPoint function, invoke it to see
+		// whether we should skip over the current user key.
+		if i.opts.SkipPoint != nil && key.Kind() != InternalKeyKindRangeKeySet && i.opts.SkipPoint(i.iterKey.UserKey) {
+			// NB: We could call nextUserKey, but in some cases the SkipPoint
+			// predicate function might be cheaper than nextUserKey's key copy
+			// and key comparison. This should be the case for MVCC suffix
+			// comparisons, for example. In the future, we could expand the
+			// SkipPoint interface to give the implementor more control over
+			// whether we skip over just the internal key, the user key, or even
+			// the key prefix.
+			i.stats.ForwardStepCount[InternalIterCall]++
+			i.iterKey, i.iterValue = i.iter.Next()
+			continue
+		}
+
 		switch key.Kind() {
 		case InternalKeyKindRangeKeySet:
 			// Save the current key.
@@ -569,7 +590,10 @@ func (i *Iterator) findNextEntry(limit []byte) {
 			i.iterValidityState = IterValid
 			return
 
-		case InternalKeyKindDelete, InternalKeyKindSingleDelete:
+		case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
+			// NB: treating InternalKeyKindSingleDelete as equivalent to DEL is not
+			// only simpler, but is also necessary for correctness due to
+			// InternalKeyKindSSTableInternalObsoleteBit.
 			i.nextUserKey()
 			continue
 
@@ -615,6 +639,13 @@ func (i *Iterator) findNextEntry(limit []byte) {
 }
 
 func (i *Iterator) nextPointCurrentUserKey() bool {
+	// If the user has configured a SkipPoint function and the current user key
+	// would be skipped by it, there's no need to step forward looking for a
+	// point key. If we were to find one, it should be skipped anyways.
+	if i.opts.SkipPoint != nil && i.opts.SkipPoint(i.key) {
+		return false
+	}
+
 	i.pos = iterPosCurForward
 
 	i.iterKey, i.iterValue = i.iter.Next()
@@ -632,7 +663,10 @@ func (i *Iterator) nextPointCurrentUserKey() bool {
 		i.err = base.CorruptionErrorf("pebble: unexpected range key set mid-user key")
 		return false
 
-	case InternalKeyKindDelete, InternalKeyKindSingleDelete:
+	case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
+		// NB: treating InternalKeyKindSingleDelete as equivalent to DEL is not
+		// only simpler, but is also necessary for correctness due to
+		// InternalKeyKindSSTableInternalObsoleteBit.
 		return false
 
 	case InternalKeyKindSet, InternalKeyKindSetWithDelete:
@@ -773,52 +807,54 @@ func (i *Iterator) maybeSampleRead() {
 func (i *Iterator) sampleRead() {
 	var topFile *manifest.FileMetadata
 	topLevel, numOverlappingLevels := numLevels, 0
-	if mi, ok := i.iter.(*mergingIter); ok {
+	mi := i.merging
-		if len(mi.levels) > 1 {
+	if mi == nil {
-			mi.ForEachLevelIter(func(li *levelIter) bool {
+		return
-				l := manifest.LevelToInt(li.level)
+	}
-				if f := li.iterFile; f != nil {
+	if len(mi.levels) > 1 {
-					var containsKey bool
+		mi.ForEachLevelIter(func(li *levelIter) bool {
-					if i.pos == iterPosNext || i.pos == iterPosCurForward ||
+			l := manifest.LevelToInt(li.level)
-						i.pos == iterPosCurForwardPaused {
+			if f := li.iterFile; f != nil {
-						containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0
+				var containsKey bool
-					} else if i.pos == iterPosPrev || i.pos == iterPosCurReverse ||
+				if i.pos == iterPosNext || i.pos == iterPosCurForward ||
-						i.pos == iterPosCurReversePaused {
+					i.pos == iterPosCurForwardPaused {
-						containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0
+					containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0
-					}
+				} else if i.pos == iterPosPrev || i.pos == iterPosCurReverse ||
-					// Do nothing if the current key is not contained in f's
+					i.pos == iterPosCurReversePaused {
-					// bounds. We could seek the LevelIterator at this level
+					containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0
-					// to find the right file, but the performance impacts of
-					// doing that are significant enough to negate the benefits
-					// of read sampling in the first place. See the discussion
-					// at:
-					// https://github.com/cockroachdb/pebble/pull/1041#issuecomment-763226492
-					if containsKey {
-						numOverlappingLevels++
-						if numOverlappingLevels >= 2 {
-							// Terminate the loop early if at least 2 overlapping levels are found.
-							return true
-						}
-						topLevel = l
-						topFile = f
-					}
 				}
-				return false
+				// Do nothing if the current key is not contained in f's
-			})
+				// bounds. We could seek the LevelIterator at this level
-		}
+				// to find the right file, but the performance impacts of
+				// doing that are significant enough to negate the benefits
+				// of read sampling in the first place. See the discussion
+				// at:
+				// https://github.com/cockroachdb/pebble/pull/1041#issuecomment-763226492
+				if containsKey {
+					numOverlappingLevels++
+					if numOverlappingLevels >= 2 {
+						// Terminate the loop early if at least 2 overlapping levels are found.
+						return true
+					}
+					topLevel = l
+					topFile = f
+				}
+			}
+			return false
+		})
 	}
 	if topFile == nil || topLevel >= numLevels {
 		return
 	}
 	if numOverlappingLevels >= 2 {
-		allowedSeeks := atomic.AddInt64(&topFile.Atomic.AllowedSeeks, -1)
+		allowedSeeks := topFile.AllowedSeeks.Add(-1)
 		if allowedSeeks == 0 {
 
 			// Since the compaction queue can handle duplicates, we can keep
 			// adding to the queue even once allowedSeeks hits 0.
 			// In fact, we NEED to keep adding to the queue, because the queue
 			// is small and evicts older and possibly useful compactions.
-			atomic.AddInt64(&topFile.Atomic.AllowedSeeks, topFile.InitAllowedSeeks)
+			topFile.AllowedSeeks.Add(topFile.InitAllowedSeeks)
 
 			read := readCompaction{
 				start:   topFile.SmallestPointKey.UserKey,
@@ -902,6 +938,26 @@ func (i *Iterator) findPrevEntry(limit []byte) {
 			}
 		}
 
+		// If the user has configured a SkipPoint function, invoke it to see
+		// whether we should skip over the current user key.
+		if i.opts.SkipPoint != nil && key.Kind() != InternalKeyKindRangeKeySet && i.opts.SkipPoint(key.UserKey) {
+			// NB: We could call prevUserKey, but in some cases the SkipPoint
+			// predicate function might be cheaper than prevUserKey's key copy
+			// and key comparison. This should be the case for MVCC suffix
+			// comparisons, for example. In the future, we could expand the
+			// SkipPoint interface to give the implementor more control over
+			// whether we skip over just the internal key, the user key, or even
+			// the key prefix.
+			i.stats.ReverseStepCount[InternalIterCall]++
+			i.iterKey, i.iterValue = i.iter.Prev()
+			if limit != nil && i.iterKey != nil && i.cmp(limit, i.iterKey.UserKey) > 0 && !i.rangeKeyWithinLimit(limit) {
+				i.iterValidityState = IterAtLimit
+				i.pos = iterPosCurReversePaused
+				return
+			}
+			continue
+		}
+
 		switch key.Kind() {
 		case InternalKeyKindRangeKeySet:
 			// Range key start boundary markers are interleaved with the maximum
@@ -930,7 +986,7 @@ func (i *Iterator) findPrevEntry(limit []byte) {
 			// return the key even if the MERGE point key is deleted.
 			rangeKeyBoundary = true
 
-		case InternalKeyKindDelete, InternalKeyKindSingleDelete:
+		case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
 			i.value = LazyValue{}
 			i.iterValidityState = IterExhausted
 			valueMerger = nil
@@ -939,12 +995,12 @@ func (i *Iterator) findPrevEntry(limit []byte) {
 			// Compare with the limit. We could optimize by only checking when
 			// we step to the previous user key, but detecting that requires a
 			// comparison too. Note that this position may already passed a
-			// number of versions of this user key, but they are all deleted,
+			// number of versions of this user key, but they are all deleted, so
-			// so the fact that a subsequent Prev*() call will not see them is
+			// the fact that a subsequent Prev*() call will not see them is
 			// harmless. Also note that this is the only place in the loop,
-			// other than the firstLoopIter case above, where we could step
+			// other than the firstLoopIter and SkipPoint cases above, where we
-			// to a different user key and start processing it for returning
+			// could step to a different user key and start processing it for
-			// to the caller.
+			// returning to the caller.
 			if limit != nil && i.iterKey != nil && i.cmp(limit, i.iterKey.UserKey) > 0 && !i.rangeKeyWithinLimit(limit) {
 				i.iterValidityState = IterAtLimit
 				i.pos = iterPosCurReversePaused
@@ -1093,9 +1149,13 @@ func (i *Iterator) mergeNext(key InternalKey, valueMerger ValueMerger) {
 			return
 		}
 		switch key.Kind() {
-		case InternalKeyKindDelete, InternalKeyKindSingleDelete:
+		case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
 			// We've hit a deletion tombstone. Return everything up to this
 			// point.
+			//
+			// NB: treating InternalKeyKindSingleDelete as equivalent to DEL is not
+			// only simpler, but is also necessary for correctness due to
+			// InternalKeyKindSSTableInternalObsoleteBit.
 			return
 
 		case InternalKeyKindSet, InternalKeyKindSetWithDelete:
@@ -1751,7 +1811,8 @@ func (i *Iterator) internalNextPrefix(currKeyPrefixLen int) {
 	i.iterKey, i.iterValue = i.iter.NextPrefix(i.prefixOrFullSeekKey)
 	if invariants.Enabled && i.iterKey != nil {
 		if iterKeyPrefixLen := i.split(i.iterKey.UserKey); i.cmp(i.iterKey.UserKey[:iterKeyPrefixLen], i.prefixOrFullSeekKey) < 0 {
-			panic("pebble: iter.NextPrefix did not advance beyond the current prefix")
+			panic(errors.AssertionFailedf("pebble: iter.NextPrefix did not advance beyond the current prefix: now at %q; expected to be geq %q",
+				i.iterKey, i.prefixOrFullSeekKey))
 		}
 	}
 }
@@ -2120,6 +2181,9 @@ func (i *Iterator) Value() []byte {
 
 // ValueAndErr returns the value, and any error encountered in extracting the value.
 // REQUIRES: i.Error()==nil and HasPointAndRange() returns true for hasPoint.
+//
+// The caller should not modify the contents of the returned slice, and its
+// contents may change on the next call to Next.
 func (i *Iterator) ValueAndErr() ([]byte, error) {
 	val, callerOwned, err := i.value.Value(i.lazyValueBuf)
 	if err != nil {
@@ -2150,7 +2214,13 @@ func (i *Iterator) RangeKeys() []RangeKeyData {
 // Valid returns true if the iterator is positioned at a valid key/value pair
 // and false otherwise.
 func (i *Iterator) Valid() bool {
-	return i.iterValidityState == IterValid && !i.requiresReposition
+	valid := i.iterValidityState == IterValid && !i.requiresReposition
+	if invariants.Enabled {
+		if err := i.Error(); valid && err != nil {
+			panic(errors.WithSecondaryError(errors.AssertionFailedf("pebble: iterator is valid with non-nil Error"), err))
+		}
+	}
+	return valid
 }
 
 // Error returns any accumulated error.
@@ -2215,6 +2285,10 @@ func (i *Iterator) Close() error {
 		i.readState = nil
 	}
 
+	if i.version != nil {
+		i.version.Unref()
+	}
+
 	for _, readers := range i.externalReaders {
 		for _, r := range readers {
 			err = firstError(err, r.Close())
@@ -2407,7 +2481,8 @@ func (i *Iterator) SetOptions(o *IterOptions) {
 	// If either options specify block property filters for an iterator stack,
 	// reconstruct it.
 	if i.pointIter != nil && (closeBoth || len(o.PointKeyFilters) > 0 || len(i.opts.PointKeyFilters) > 0 ||
-		o.RangeKeyMasking.Filter != nil || i.opts.RangeKeyMasking.Filter != nil) {
+		o.RangeKeyMasking.Filter != nil || i.opts.RangeKeyMasking.Filter != nil || o.SkipPoint != nil ||
+		i.opts.SkipPoint != nil) {
 		i.err = firstError(i.err, i.pointIter.Close())
 		i.pointIter = nil
 	}
@@ -2646,11 +2721,22 @@ func (i *Iterator) CloneWithContext(ctx context.Context, opts CloneOptions) (*It
 	}
 
 	readState := i.readState
-	if readState == nil {
+	vers := i.version
+	if readState == nil && vers == nil {
 		return nil, errors.Errorf("cannot Clone a closed Iterator")
 	}
 	// i is already holding a ref, so there is no race with unref here.
-	readState.ref()
+	//
+	// TODO(bilal): If the underlying iterator was created on a snapshot, we could
+	// grab a reference to the current readState instead of reffing the original
+	// readState. This allows us to release references to some zombie sstables
+	// and memtables.
+	if readState != nil {
+		readState.ref()
+	}
+	if vers != nil {
+		vers.Ref()
+	}
 	// Bundle various structures under a single umbrella in order to allocate
 	// them together.
 	buf := iterAllocPool.Get().(*iterAlloc)
@@ -2662,6 +2748,7 @@ func (i *Iterator) CloneWithContext(ctx context.Context, opts CloneOptions) (*It
 		merge:               i.merge,
 		comparer:            i.comparer,
 		readState:           readState,
+		version:             vers,
 		keyBuf:              buf.keyBuf,
 		prefixOrFullSeekKey: buf.prefixOrFullSeekKey,
 		boundsBuf:           buf.boundsBuf,
@@ -2716,19 +2803,19 @@ func (stats *IteratorStats) SafeFormat(s redact.SafePrinter, verb rune) {
 		s.SafeString(",\n(internal-stats: ")
 		s.Printf("(block-bytes: (total %s, cached %s, read-time %s)), "+
 			"(points: (count %s, key-bytes %s, value-bytes %s, tombstoned %s))",
-			humanize.IEC.Uint64(stats.InternalStats.BlockBytes),
+			humanize.Bytes.Uint64(stats.InternalStats.BlockBytes),
-			humanize.IEC.Uint64(stats.InternalStats.BlockBytesInCache),
+			humanize.Bytes.Uint64(stats.InternalStats.BlockBytesInCache),
 			humanize.FormattedString(stats.InternalStats.BlockReadDuration.String()),
-			humanize.SI.Uint64(stats.InternalStats.PointCount),
+			humanize.Count.Uint64(stats.InternalStats.PointCount),
-			humanize.SI.Uint64(stats.InternalStats.KeyBytes),
+			humanize.Bytes.Uint64(stats.InternalStats.KeyBytes),
-			humanize.SI.Uint64(stats.InternalStats.ValueBytes),
+			humanize.Bytes.Uint64(stats.InternalStats.ValueBytes),
-			humanize.SI.Uint64(stats.InternalStats.PointsCoveredByRangeTombstones),
+			humanize.Count.Uint64(stats.InternalStats.PointsCoveredByRangeTombstones),
 		)
 		if stats.InternalStats.SeparatedPointValue.Count != 0 {
 			s.Printf(", (separated: (count %s, bytes %s, fetched %s)))",
-				humanize.SI.Uint64(stats.InternalStats.SeparatedPointValue.Count),
+				humanize.Count.Uint64(stats.InternalStats.SeparatedPointValue.Count),
-				humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytes),
+				humanize.Bytes.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytes),
-				humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytesFetched))
+				humanize.Bytes.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytesFetched))
 		} else {
 			s.Printf(")")
 		}
@@ -2741,3 +2828,169 @@ func (stats *IteratorStats) SafeFormat(s redact.SafePrinter, verb rune) {
 			stats.RangeKeyStats.SkippedPoints)
 	}
 }
+
+// CanDeterministicallySingleDelete takes a valid iterator and examines internal
+// state to determine if a SingleDelete deleting Iterator.Key() would
+// deterministically delete the key. CanDeterministicallySingleDelete requires
+// the iterator to be oriented in the forward direction (eg, the last
+// positioning operation must've been a First, a Seek[Prefix]GE, or a
+// Next[Prefix][WithLimit]).
+//
+// This function does not change the external position of the iterator, and all
+// positioning methods should behave the same as if it was never called. This
+// function will only return a meaningful result the first time it's invoked at
+// an iterator position. This function invalidates the iterator Value's memory,
+// and the caller must not rely on the memory safety of the previous Iterator
+// position.
+//
+// If CanDeterministicallySingleDelete returns true AND the key at the iterator
+// position is not modified between the creation of the Iterator and the commit
+// of a batch containing a SingleDelete over the key, then the caller can be
+// assured that SingleDelete is equivalent to Delete on the local engine, but it
+// may not be true on another engine that received the same writes and with
+// logically equivalent state since this engine may have collapsed multiple SETs
+// into one.
+func CanDeterministicallySingleDelete(it *Iterator) (bool, error) {
+	// This function may only be called once per external iterator position. We
+	// can validate this by checking the last positioning operation.
+	if it.lastPositioningOp == internalNextOp {
+		return false, errors.New("pebble: CanDeterministicallySingleDelete called twice")
+	}
+	validity, kind := it.internalNext()
+	var shadowedBySingleDelete bool
+	for validity == internalNextValid {
+		switch kind {
+		case InternalKeyKindDelete, InternalKeyKindDeleteSized:
+			// A DEL or DELSIZED tombstone is okay. An internal key
+			// sequence like SINGLEDEL; SET; DEL; SET can be handled
+			// deterministically. If there are SETs further down, we
+			// don't care about them.
+			return true, nil
+		case InternalKeyKindSingleDelete:
+			// A SingleDelete is okay as long as when that SingleDelete was
+			// written, it was written deterministically (eg, with its own
+			// CanDeterministicallySingleDelete check). Validate that it was
+			// written deterministically. We'll allow one set to appear after
+			// the SingleDelete.
+			shadowedBySingleDelete = true
+			validity, kind = it.internalNext()
+			continue
+		case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
+			// If we observed a single delete, it's allowed to delete 1 key.
+			// We'll keep looping to validate that the internal keys beneath the
+			// already-written single delete are copacetic.
+			if shadowedBySingleDelete {
+				shadowedBySingleDelete = false
+				validity, kind = it.internalNext()
+				continue
+			}
+			// We encountered a shadowed SET, SETWITHDEL, MERGE. A SINGLEDEL
+			// that deleted the KV at the original iterator position could
+			// result in this key becoming visible.
+			return false, nil
+		case InternalKeyKindRangeDelete:
+			// RangeDeletes are handled by the merging iterator and should never
+			// be observed by the top-level Iterator.
+			panic(errors.AssertionFailedf("pebble: unexpected range delete"))
+		case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
+			// Range keys are interleaved at the maximal sequence number and
+			// should never be observed within a user key.
+			panic(errors.AssertionFailedf("pebble: unexpected range key"))
+		default:
+			panic(errors.AssertionFailedf("pebble: unexpected key kind: %s", errors.Safe(kind)))
+		}
+	}
+	if validity == internalNextError {
+		return false, it.Error()
+	}
+	return true, nil
+}
+
+// internalNextValidity enumerates the potential outcomes of a call to
+// internalNext.
+type internalNextValidity int8
+
+const (
+	// internalNextError is returned by internalNext when an error occurred and
+	// the caller is responsible for checking iter.Error().
+	internalNextError internalNextValidity = iota
+	// internalNextExhausted is returned by internalNext when the next internal
+	// key is an internal key with a different user key than Iterator.Key().
+	internalNextExhausted
+	// internalNextValid is returned by internalNext when the internal next
+	// found a shadowed internal key with a user key equal to Iterator.Key().
+	internalNextValid
+)
+
+// internalNext advances internal Iterator state forward to expose the
+// InternalKeyKind of the next internal key with a user key equal to Key().
+//
+// internalNext is a highly specialized operation and is unlikely to be
+// generally useful. See Iterator.Next for how to reposition the iterator to the
+// next key. internalNext requires the Iterator to be at a valid position in the
+// forward direction (the last positioning operation must've been a First, a
+// Seek[Prefix]GE, or a Next[Prefix][WithLimit] and Valid() must return true).
+//
+// internalNext, unlike all other Iterator methods, exposes internal LSM state.
+// internalNext advances the Iterator's internal iterator to the next shadowed
+// key with a user key equal to Key(). When a key is overwritten or deleted, its
+// removal from the LSM occurs lazily as a part of compactions. internalNext
+// allows the caller to see whether an obsolete internal key exists with the
+// current Key(), and what it's key kind is. Note that the existence of an
+// internal key is nondeterministic and dependent on internal LSM state. These
+// semantics are unlikely to be applicable to almost all use cases.
+//
+// If internalNext finds a key that shares the same user key as Key(), it
+// returns internalNextValid and the internal key's kind. If internalNext
+// encounters an error, it returns internalNextError and the caller is expected
+// to call Iterator.Error() to retrieve it. In all other circumstances,
+// internalNext returns internalNextExhausted, indicating that there are no more
+// additional internal keys with the user key Key().
+//
+// internalNext does not change the external position of the iterator, and a
+// Next operation should behave the same as if internalNext was never called.
+// internalNext does invalidate the iterator Value's memory, and the caller must
+// not rely on the memory safety of the previous Iterator position.
+func (i *Iterator) internalNext() (internalNextValidity, base.InternalKeyKind) {
+	i.stats.ForwardStepCount[InterfaceCall]++
+	if i.err != nil {
+		return internalNextError, base.InternalKeyKindInvalid
+	} else if i.iterValidityState != IterValid {
+		return internalNextExhausted, base.InternalKeyKindInvalid
+	}
+	i.lastPositioningOp = internalNextOp
+
+	switch i.pos {
+	case iterPosCurForward:
+		i.iterKey, i.iterValue = i.iter.Next()
+		if i.iterKey == nil {
+			// We check i.iter.Error() here and return an internalNextError enum
+			// variant so that the caller does not need to check i.iter.Error()
+			// in the common case that the next internal key has a new user key.
+			if i.err = i.iter.Error(); i.err != nil {
+				return internalNextError, base.InternalKeyKindInvalid
+			}
+			i.pos = iterPosNext
+			return internalNextExhausted, base.InternalKeyKindInvalid
+		} else if i.comparer.Equal(i.iterKey.UserKey, i.key) {
+			return internalNextValid, i.iterKey.Kind()
+		}
+		i.pos = iterPosNext
+		return internalNextExhausted, base.InternalKeyKindInvalid
+	case iterPosCurReverse, iterPosCurReversePaused, iterPosPrev:
+		i.err = errors.New("switching from reverse to forward via internalNext is prohibited")
+		i.iterValidityState = IterExhausted
+		return internalNextError, base.InternalKeyKindInvalid
+	case iterPosNext, iterPosCurForwardPaused:
+		// The previous method already moved onto the next user key. This is
+		// only possible if
+		//   - the last positioning method was a call to internalNext, and we
+		//     advanced to a new user key.
+		//   - the previous non-internalNext iterator operation encountered a
+		//     range key or merge, forcing an internal Next that found a new
+		//     user key that's not equal to i.Iterator.Key().
+		return internalNextExhausted, base.InternalKeyKindInvalid
+	default:
+		panic("unreachable")
+	}
+}

vendor/github.com/cockroachdb/pebble/level_checker.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"io"
 	"sort"
-	"sync/atomic"
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
@@ -166,7 +165,7 @@ func (m *simpleMergingIter) step() bool {
 		if m.valueMerger != nil {
 			// Ongoing series of MERGE records.
 			switch item.key.Kind() {
-			case InternalKeyKindSingleDelete, InternalKeyKindDelete:
+			case InternalKeyKindSingleDelete, InternalKeyKindDelete, InternalKeyKindDeleteSized:
 				var closer io.Closer
 				_, closer, m.err = m.valueMerger.Finish(true /* includesBase */)
 				if m.err == nil && closer != nil {
@@ -346,7 +345,7 @@ func iterateAndCheckTombstones(
 
 type checkConfig struct {
 	logger    Logger
-	cmp       Compare
+	comparer  *Comparer
 	readState *readState
 	newIters  tableNewIters
 	seqNum    uint64
@@ -355,6 +354,9 @@ type checkConfig struct {
 	formatKey base.FormatKey
 }
 
+// cmp is shorthand for comparer.Compare.
+func (c *checkConfig) cmp(a, b []byte) int { return c.comparer.Compare(a, b) }
+
 func checkRangeTombstones(c *checkConfig) error {
 	var level int
 	var tombstones []tombstoneWithLevel
@@ -380,7 +382,7 @@ func checkRangeTombstones(c *checkConfig) error {
 			atomicUnit, _ := expandToAtomicUnit(c.cmp, lf.Slice(), true /* disableIsCompacting */)
 			lower, upper := manifest.KeyRange(c.cmp, atomicUnit.Iter())
 			iterToClose, iter, err := c.newIters(
-				context.Background(), lf.FileMetadata, nil, internalIterOpts{})
+				context.Background(), lf.FileMetadata, &IterOptions{level: manifest.Level(lsmLevel)}, internalIterOpts{})
 			if err != nil {
 				return err
 			}
@@ -568,11 +570,11 @@ func (d *DB) CheckLevels(stats *CheckLevelsStats) error {
 
 	// Determine the seqnum to read at after grabbing the read state (current and
 	// memtables) above.
-	seqNum := atomic.LoadUint64(&d.mu.versions.atomic.visibleSeqNum)
+	seqNum := d.mu.versions.visibleSeqNum.Load()
 
 	checkConfig := &checkConfig{
 		logger:    d.opts.Logger,
-		cmp:       d.cmp,
+		comparer:  d.opts.Comparer,
 		readState: readState,
 		newIters:  d.newIters,
 		seqNum:    seqNum,
@@ -640,7 +642,7 @@ func checkLevelsInternal(c *checkConfig) (err error) {
 		manifestIter := current.L0SublevelFiles[sublevel].Iter()
 		iterOpts := IterOptions{logger: c.logger}
 		li := &levelIter{}
-		li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters, manifestIter,
+		li.init(context.Background(), iterOpts, c.comparer, c.newIters, manifestIter,
 			manifest.L0Sublevel(sublevel), internalIterOpts{})
 		li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
 		li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext)
@@ -654,7 +656,7 @@ func checkLevelsInternal(c *checkConfig) (err error) {
 
 		iterOpts := IterOptions{logger: c.logger}
 		li := &levelIter{}
-		li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters,
+		li.init(context.Background(), iterOpts, c.comparer, c.newIters,
 			current.Levels[level].Iter(), manifest.Level(level), internalIterOpts{})
 		li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
 		li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext)

vendor/github.com/cockroachdb/pebble/level_iter.go

@@ -36,9 +36,8 @@ type tableNewIters func(
 // tableNewRangeDelIter takes a tableNewIters and returns a TableNewSpanIter
 // for the rangedel iterator returned by tableNewIters.
 func tableNewRangeDelIter(ctx context.Context, newIters tableNewIters) keyspan.TableNewSpanIter {
-	return func(file *manifest.FileMetadata, iterOptions *keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) {
+	return func(file *manifest.FileMetadata, iterOptions keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) {
-		iter, rangeDelIter, err := newIters(
+		iter, rangeDelIter, err := newIters(ctx, file, nil, internalIterOpts{})
-			ctx, file, &IterOptions{RangeKeyFilters: iterOptions.RangeKeyFilters}, internalIterOpts{})
 		if iter != nil {
 			_ = iter.Close()
 		}
@@ -51,6 +50,7 @@ func tableNewRangeDelIter(ctx context.Context, newIters tableNewIters) keyspan.T
 
 type internalIterOpts struct {
 	bytesIterated      *uint64
+	bufferPool         *sstable.BufferPool
 	stats              *base.InternalIteratorStats
 	boundLimitedFilter sstable.BoundLimitedBlockPropertyFilter
 }
@@ -82,10 +82,11 @@ type levelIter struct {
 	// short-lived (since they pin sstables), (b) plumbing a context into every
 	// method is very painful, (c) they do not (yet) respect context
 	// cancellation and are only used for tracing.
-	ctx    context.Context
+	ctx      context.Context
-	logger Logger
+	logger   Logger
-	cmp    Compare
+	comparer *Comparer
-	split  Split
+	cmp      Compare
+	split    Split
 	// The lower/upper bounds for iteration as specified at creation or the most
 	// recent call to SetBounds.
 	lower []byte
@@ -200,6 +201,11 @@ type levelIter struct {
 	// cache when constructing new table iterators.
 	internalOpts internalIterOpts
 
+	// Scratch space for the obsolete keys filter, when there are no other block
+	// property filters specified. See the performance note where
+	// IterOptions.PointKeyFilters is declared.
+	filtersBuf [1]BlockPropertyFilter
+
 	// Disable invariant checks even if they are otherwise enabled. Used by tests
 	// which construct "impossible" situations (e.g. seeking to a key before the
 	// lower bound).
@@ -236,24 +242,22 @@ var _ base.InternalIterator = (*levelIter)(nil)
 // parameter if the caller is never going to call SeekPrefixGE.
 func newLevelIter(
 	opts IterOptions,
-	cmp Compare,
+	comparer *Comparer,
-	split Split,
 	newIters tableNewIters,
 	files manifest.LevelIterator,
 	level manifest.Level,
-	bytesIterated *uint64,
+	internalOpts internalIterOpts,
 ) *levelIter {
 	l := &levelIter{}
-	l.init(context.Background(), opts, cmp, split, newIters, files, level,
+	l.init(context.Background(), opts, comparer, newIters, files, level,
-		internalIterOpts{bytesIterated: bytesIterated})
+		internalOpts)
 	return l
 }
 
 func (l *levelIter) init(
 	ctx context.Context,
 	opts IterOptions,
-	cmp Compare,
+	comparer *Comparer,
-	split Split,
 	newIters tableNewIters,
 	files manifest.LevelIterator,
 	level manifest.Level,
@@ -267,10 +271,15 @@ func (l *levelIter) init(
 	l.upper = opts.UpperBound
 	l.tableOpts.TableFilter = opts.TableFilter
 	l.tableOpts.PointKeyFilters = opts.PointKeyFilters
+	if len(opts.PointKeyFilters) == 0 {
+		l.tableOpts.PointKeyFilters = l.filtersBuf[:0:1]
+	}
 	l.tableOpts.UseL6Filters = opts.UseL6Filters
 	l.tableOpts.level = l.level
-	l.cmp = cmp
+	l.tableOpts.snapshotForHideObsoletePoints = opts.snapshotForHideObsoletePoints
-	l.split = split
+	l.comparer = comparer
+	l.cmp = comparer.Compare
+	l.split = comparer.Split
 	l.iterFile = nil
 	l.newIters = newIters
 	l.files = files
@@ -788,7 +797,7 @@ func (l *levelIter) SeekPrefixGE(
 		}
 		return l.verify(l.largestBoundary, base.LazyValue{})
 	}
-	// It is possible that we are here because bloom filter matching failed.  In
+	// It is possible that we are here because bloom filter matching failed. In
 	// that case it is likely that all keys matching the prefix are wholly
 	// within the current file and cannot be in the subsequent file. In that
 	// case we don't want to go to the next file, since loading and seeking in
@@ -796,7 +805,16 @@ func (l *levelIter) SeekPrefixGE(
 	// next file will defeat the optimization for the next SeekPrefixGE that is
 	// called with flags.TrySeekUsingNext(), since for sparse key spaces it is
 	// likely that the next key will also be contained in the current file.
-	if n := l.split(l.iterFile.LargestPointKey.UserKey); l.cmp(prefix, l.iterFile.LargestPointKey.UserKey[:n]) < 0 {
+	var n int
+	if l.split != nil {
+		// If the split function is specified, calculate the prefix length accordingly.
+		n = l.split(l.iterFile.LargestPointKey.UserKey)
+	} else {
+		// If the split function is not specified, the entire key is used as the
+		// prefix. This case can occur when getIter uses SeekPrefixGE.
+		n = len(l.iterFile.LargestPointKey.UserKey)
+	}
+	if l.cmp(prefix, l.iterFile.LargestPointKey.UserKey[:n]) < 0 {
 		return nil, base.LazyValue{}
 	}
 	return l.verify(l.skipEmptyFileForward())
@@ -1066,7 +1084,9 @@ func (l *levelIter) skipEmptyFileForward() (*InternalKey, base.LazyValue) {
 			if *l.rangeDelIterPtr != nil && l.filteredIter != nil &&
 				l.filteredIter.MaybeFilteredKeys() {
 				l.largestBoundary = &l.iterFile.Largest
-				l.boundaryContext.isIgnorableBoundaryKey = true
+				if l.boundaryContext != nil {
+					l.boundaryContext.isIgnorableBoundaryKey = true
+				}
 				return l.largestBoundary, base.LazyValue{}
 			}
 		}
@@ -1155,7 +1175,9 @@ func (l *levelIter) skipEmptyFileBackward() (*InternalKey, base.LazyValue) {
 			// the next file.
 			if *l.rangeDelIterPtr != nil && l.filteredIter != nil && l.filteredIter.MaybeFilteredKeys() {
 				l.smallestBoundary = &l.iterFile.Smallest
-				l.boundaryContext.isIgnorableBoundaryKey = true
+				if l.boundaryContext != nil {
+					l.boundaryContext.isIgnorableBoundaryKey = true
+				}
 				return l.smallestBoundary, base.LazyValue{}
 			}
 		}

vendor/github.com/cockroachdb/pebble/log_recycler.go

@@ -32,14 +32,14 @@ type logRecycler struct {
 // the log file should not be deleted (i.e. the log is being recycled), and
 // false otherwise.
 func (r *logRecycler) add(logInfo fileInfo) bool {
-	if logInfo.fileNum < r.minRecycleLogNum {
+	if logInfo.fileNum.FileNum() < r.minRecycleLogNum {
 		return false
 	}
 
 	r.mu.Lock()
 	defer r.mu.Unlock()
 
-	if logInfo.fileNum <= r.mu.maxLogNum {
+	if logInfo.fileNum.FileNum() <= r.mu.maxLogNum {
 		// The log file number was already considered for recycling. Don't consider
 		// it again. This avoids a race between adding the same log file for
 		// recycling multiple times, and removing the log file for actual
@@ -49,7 +49,7 @@ func (r *logRecycler) add(logInfo fileInfo) bool {
 		// shouldn't be deleted.
 		return true
 	}
-	r.mu.maxLogNum = logInfo.fileNum
+	r.mu.maxLogNum = logInfo.fileNum.FileNum()
 	if len(r.mu.logs) >= r.limit {
 		return false
 	}
@@ -89,7 +89,7 @@ func (r *logRecycler) pop(logNum FileNum) error {
 	if len(r.mu.logs) == 0 {
 		return errors.New("pebble: log recycler empty")
 	}
-	if r.mu.logs[0].fileNum != logNum {
+	if r.mu.logs[0].fileNum.FileNum() != logNum {
 		return errors.Errorf("pebble: log recycler invalid %d vs %d", errors.Safe(logNum), errors.Safe(fileInfoNums(r.mu.logs)))
 	}
 	r.mu.logs = r.mu.logs[1:]
@@ -102,7 +102,7 @@ func fileInfoNums(finfos []fileInfo) []FileNum {
 	}
 	nums := make([]FileNum, len(finfos))
 	for i := range finfos {
-		nums[i] = finfos[i].fileNum
+		nums[i] = finfos[i].fileNum.FileNum()
 	}
 	return nums
 }

vendor/github.com/cockroachdb/pebble/mem_table.go

@@ -10,12 +10,12 @@ import (
 	"os"
 	"sync"
 	"sync/atomic"
-	"unsafe"
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/arenaskl"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/keyspan"
+	"github.com/cockroachdb/pebble/internal/manual"
 	"github.com/cockroachdb/pebble/internal/rangedel"
 	"github.com/cockroachdb/pebble/internal/rangekey"
 )
@@ -80,12 +80,21 @@ type memTable struct {
 	// inflight mutations that have reserved space in the memtable but not yet
 	// applied. The memtable cannot be flushed to disk until the writer refs
 	// drops to zero.
-	writerRefs int32
+	writerRefs atomic.Int32
 	tombstones keySpanCache
 	rangeKeys  keySpanCache
 	// The current logSeqNum at the time the memtable was created. This is
 	// guaranteed to be less than or equal to any seqnum stored in the memtable.
-	logSeqNum uint64
+	logSeqNum                    uint64
+	releaseAccountingReservation func()
+}
+
+func (m *memTable) free() {
+	if m != nil {
+		m.releaseAccountingReservation()
+		manual.Free(m.arenaBuf)
+		m.arenaBuf = nil
+	}
 }
 
 // memTableOptions holds configuration used when creating a memTable. All of
@@ -93,9 +102,10 @@ type memTable struct {
 // which is used by tests.
 type memTableOptions struct {
 	*Options
-	arenaBuf  []byte
+	arenaBuf                     []byte
-	size      int
+	size                         int
-	logSeqNum uint64
+	logSeqNum                    uint64
+	releaseAccountingReservation func()
 }
 
 func checkMemTable(obj interface{}) {
@@ -110,18 +120,24 @@ func checkMemTable(obj interface{}) {
 // Options.MemTableSize is used instead.
 func newMemTable(opts memTableOptions) *memTable {
 	opts.Options = opts.Options.EnsureDefaults()
-	if opts.size == 0 {
+	m := new(memTable)
-		opts.size = opts.MemTableSize
+	m.init(opts)
-	}
+	return m
+}
 
-	m := &memTable{
+func (m *memTable) init(opts memTableOptions) {
-		cmp:        opts.Comparer.Compare,
+	if opts.size == 0 {
-		formatKey:  opts.Comparer.FormatKey,
+		opts.size = int(opts.MemTableSize)
-		equal:      opts.Comparer.Equal,
-		arenaBuf:   opts.arenaBuf,
-		writerRefs: 1,
-		logSeqNum:  opts.logSeqNum,
 	}
+	*m = memTable{
+		cmp:                          opts.Comparer.Compare,
+		formatKey:                    opts.Comparer.FormatKey,
+		equal:                        opts.Comparer.Equal,
+		arenaBuf:                     opts.arenaBuf,
+		logSeqNum:                    opts.logSeqNum,
+		releaseAccountingReservation: opts.releaseAccountingReservation,
+	}
+	m.writerRefs.Store(1)
 	m.tombstones = keySpanCache{
 		cmp:           m.cmp,
 		formatKey:     m.formatKey,
@@ -144,18 +160,18 @@ func newMemTable(opts memTableOptions) *memTable {
 	m.rangeDelSkl.Reset(arena, m.cmp)
 	m.rangeKeySkl.Reset(arena, m.cmp)
 	m.reserved = arena.Size()
-	return m
 }
 
 func (m *memTable) writerRef() {
-	switch v := atomic.AddInt32(&m.writerRefs, 1); {
+	switch v := m.writerRefs.Add(1); {
 	case v <= 1:
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
 	}
 }
 
-func (m *memTable) writerUnref() bool {
+// writerUnref drops a ref on the memtable. Returns true if this was the last ref.
-	switch v := atomic.AddInt32(&m.writerRefs, -1); {
+func (m *memTable) writerUnref() (wasLastRef bool) {
+	switch v := m.writerRefs.Add(-1); {
 	case v < 0:
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
 	case v == 0:
@@ -165,8 +181,9 @@ func (m *memTable) writerUnref() bool {
 	}
 }
 
+// readyForFlush is part of the flushable interface.
 func (m *memTable) readyForFlush() bool {
-	return atomic.LoadInt32(&m.writerRefs) == 0
+	return m.writerRefs.Load() == 0
 }
 
 // Prepare reserves space for the batch in the memtable and references the
@@ -194,11 +211,13 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error {
 	var tombstoneCount, rangeKeyCount uint32
 	startSeqNum := seqNum
 	for r := batch.Reader(); ; seqNum++ {
-		kind, ukey, value, ok := r.Next()
+		kind, ukey, value, ok, err := r.Next()
 		if !ok {
+			if err != nil {
+				return err
+			}
 			break
 		}
-		var err error
 		ikey := base.MakeInternalKey(ukey, seqNum, kind)
 		switch kind {
 		case InternalKeyKindRangeDelete:
@@ -233,17 +252,19 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error {
 	return nil
 }
 
-// newIter returns an iterator that is unpositioned (Iterator.Valid() will
+// newIter is part of the flushable interface. It returns an iterator that is
-// return false). The iterator can be positioned via a call to SeekGE,
+// unpositioned (Iterator.Valid() will return false). The iterator can be
-// SeekLT, First or Last.
+// positioned via a call to SeekGE, SeekLT, First or Last.
 func (m *memTable) newIter(o *IterOptions) internalIterator {
 	return m.skl.NewIter(o.GetLowerBound(), o.GetUpperBound())
 }
 
+// newFlushIter is part of the flushable interface.
 func (m *memTable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
 	return m.skl.NewFlushIter(bytesFlushed)
 }
 
+// newRangeDelIter is part of the flushable interface.
 func (m *memTable) newRangeDelIter(*IterOptions) keyspan.FragmentIterator {
 	tombstones := m.tombstones.get()
 	if tombstones == nil {
@@ -252,6 +273,7 @@ func (m *memTable) newRangeDelIter(*IterOptions) keyspan.FragmentIterator {
 	return keyspan.NewIter(m.cmp, tombstones)
 }
 
+// newRangeKeyIter is part of the flushable interface.
 func (m *memTable) newRangeKeyIter(*IterOptions) keyspan.FragmentIterator {
 	rangeKeys := m.rangeKeys.get()
 	if rangeKeys == nil {
@@ -260,13 +282,14 @@ func (m *memTable) newRangeKeyIter(*IterOptions) keyspan.FragmentIterator {
 	return keyspan.NewIter(m.cmp, rangeKeys)
 }
 
+// containsRangeKeys is part of the flushable interface.
 func (m *memTable) containsRangeKeys() bool {
-	return atomic.LoadUint32(&m.rangeKeys.atomicCount) > 0
+	return m.rangeKeys.count.Load() > 0
 }
 
 func (m *memTable) availBytes() uint32 {
 	a := m.skl.Arena()
-	if atomic.LoadInt32(&m.writerRefs) == 1 {
+	if m.writerRefs.Load() == 1 {
 		// If there are no other concurrent apply operations, we can update the
 		// reserved bytes setting to accurately reflect how many bytes of been
 		// allocated vs the over-estimation present in memTableEntrySize.
@@ -275,10 +298,12 @@ func (m *memTable) availBytes() uint32 {
 	return a.Capacity() - m.reserved
 }
 
+// inuseBytes is part of the flushable interface.
 func (m *memTable) inuseBytes() uint64 {
 	return uint64(m.skl.Size() - memTableEmptySize)
 }
 
+// totalBytes is part of the flushable interface.
 func (m *memTable) totalBytes() uint64 {
 	return uint64(m.skl.Arena().Capacity())
 }
@@ -355,8 +380,8 @@ func (f *keySpanFrags) get(
 // invalidated whenever a key of the same kind is added to a memTable, and
 // populated when empty when a span iterator of that key kind is created.
 type keySpanCache struct {
-	atomicCount   uint32
+	count         atomic.Uint32
-	frags         unsafe.Pointer
+	frags         atomic.Pointer[keySpanFrags]
 	cmp           Compare
 	formatKey     base.FormatKey
 	constructSpan constructSpan
@@ -366,23 +391,20 @@ type keySpanCache struct {
 // Invalidate the current set of cached spans, indicating the number of
 // spans that were added.
 func (c *keySpanCache) invalidate(count uint32) {
-	newCount := atomic.AddUint32(&c.atomicCount, count)
+	newCount := c.count.Add(count)
 	var frags *keySpanFrags
 
 	for {
-		oldPtr := atomic.LoadPointer(&c.frags)
+		oldFrags := c.frags.Load()
-		if oldPtr != nil {
+		if oldFrags != nil && oldFrags.count >= newCount {
-			oldFrags := (*keySpanFrags)(oldPtr)
+			// Someone else invalidated the cache before us and their invalidation
-			if oldFrags.count >= newCount {
+			// subsumes ours.
-				// Someone else invalidated the cache before us and their invalidation
+			break
-				// subsumes ours.
-				break
-			}
 		}
 		if frags == nil {
 			frags = &keySpanFrags{count: newCount}
 		}
-		if atomic.CompareAndSwapPointer(&c.frags, oldPtr, unsafe.Pointer(frags)) {
+		if c.frags.CompareAndSwap(oldFrags, frags) {
 			// We successfully invalidated the cache.
 			break
 		}
@@ -391,7 +413,7 @@ func (c *keySpanCache) invalidate(count uint32) {
 }
 
 func (c *keySpanCache) get() []keyspan.Span {
-	frags := (*keySpanFrags)(atomic.LoadPointer(&c.frags))
+	frags := c.frags.Load()
 	if frags == nil {
 		return nil
 	}

vendor/github.com/cockroachdb/pebble/merging_iter.go

@@ -27,6 +27,10 @@ type mergingIterLevel struct {
 	// iterKey and iterValue cache the current key and value iter are pointed at.
 	iterKey   *InternalKey
 	iterValue base.LazyValue
+	// levelIter is non-nil if this level's iter is ultimately backed by a
+	// *levelIter. The handle in iter may have wrapped the levelIter with
+	// intermediary internalIterator implementations.
+	levelIter *levelIter
 
 	// levelIterBoundaryContext's fields are set when using levelIter, in order
 	// to surface sstable boundary keys and file-level context. See levelIter
@@ -1371,9 +1375,9 @@ func (m *mergingIter) DebugString() string {
 }
 
 func (m *mergingIter) ForEachLevelIter(fn func(li *levelIter) bool) {
-	for _, iter := range m.levels {
+	for _, ml := range m.levels {
-		if li, ok := iter.iter.(*levelIter); ok {
+		if ml.levelIter != nil {
-			if done := fn(li); done {
+			if done := fn(ml.levelIter); done {
 				break
 			}
 		}

vendor/github.com/cockroachdb/pebble/metrics.go

@@ -6,12 +6,13 @@ package pebble
 
 import (
 	"fmt"
-	"strings"
+	"math"
 	"time"
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/cockroachdb/pebble/internal/humanize"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
 	"github.com/cockroachdb/pebble/record"
 	"github.com/cockroachdb/pebble/sstable"
 	"github.com/cockroachdb/redact"
@@ -28,13 +29,10 @@ type FilterMetrics = sstable.FilterMetrics
 // comment in base.
 type ThroughputMetric = base.ThroughputMetric
 
-func formatCacheMetrics(w redact.SafePrinter, m *CacheMetrics, name redact.SafeString) {
+// SecondaryCacheMetrics holds metrics for the persistent secondary cache
-	w.Printf("%7s %9s %7s %6.1f%%  (score == hit-rate)\n",
+// that caches commonly accessed blocks from blob storage on a local
-		name,
+// file system.
-		humanize.SI.Int64(m.Count),
+type SecondaryCacheMetrics = sharedcache.Metrics
-		humanize.IEC.Int64(m.Size),
-		redact.Safe(hitRate(m.Hits, m.Misses)))
-}
 
 // LevelMetrics holds per-level metrics such as the number of files and total
 // size of the files, and compaction related metrics.
@@ -46,9 +44,14 @@ type LevelMetrics struct {
 	Sublevels int32
 	// The total number of files in the level.
 	NumFiles int64
+	// The total number of virtual sstables in the level.
+	NumVirtualFiles uint64
 	// The total size in bytes of the files in the level.
 	Size int64
-	// The level's compaction score.
+	// The total size of the virtual sstables in the level.
+	VirtualSize uint64
+	// The level's compaction score. This is the compensatedScoreRatio in the
+	// candidateLevelInfo.
 	Score float64
 	// The number of incoming bytes from other levels read during
 	// compactions. This excludes bytes moved and bytes ingested. For L0 this is
@@ -79,6 +82,18 @@ type LevelMetrics struct {
 	TablesIngested uint64
 	// The number of sstables moved to this level by a "move" compaction.
 	TablesMoved uint64
+
+	MultiLevel struct {
+		// BytesInTop are the total bytes in a multilevel compaction coming from the top level.
+		BytesInTop uint64
+
+		// BytesIn, exclusively for multiLevel compactions.
+		BytesIn uint64
+
+		// BytesRead, exclusively for multilevel compactions.
+		BytesRead uint64
+	}
+
 	// Additional contains misc additional metrics that are not always printed.
 	Additional struct {
 		// The sum of Properties.ValueBlocksSize for all the sstables in this
@@ -96,6 +111,8 @@ type LevelMetrics struct {
 // Add updates the counter metrics for the level.
 func (m *LevelMetrics) Add(u *LevelMetrics) {
 	m.NumFiles += u.NumFiles
+	m.NumVirtualFiles += u.NumVirtualFiles
+	m.VirtualSize += u.VirtualSize
 	m.Size += u.Size
 	m.BytesIn += u.BytesIn
 	m.BytesIngested += u.BytesIngested
@@ -107,6 +124,9 @@ func (m *LevelMetrics) Add(u *LevelMetrics) {
 	m.TablesFlushed += u.TablesFlushed
 	m.TablesIngested += u.TablesIngested
 	m.TablesMoved += u.TablesMoved
+	m.MultiLevel.BytesInTop += u.MultiLevel.BytesInTop
+	m.MultiLevel.BytesRead += u.MultiLevel.BytesRead
+	m.MultiLevel.BytesIn += u.MultiLevel.BytesIn
 	m.Additional.BytesWrittenDataBlocks += u.Additional.BytesWrittenDataBlocks
 	m.Additional.BytesWrittenValueBlocks += u.Additional.BytesWrittenValueBlocks
 	m.Additional.ValueBlocksSize += u.Additional.ValueBlocksSize
@@ -121,32 +141,6 @@ func (m *LevelMetrics) WriteAmp() float64 {
 	return float64(m.BytesFlushed+m.BytesCompacted) / float64(m.BytesIn)
 }
 
-// format generates a string of the receiver's metrics, formatting it into the
-// supplied buffer.
-func (m *LevelMetrics) format(
-	w redact.SafePrinter, score redact.SafeValue, includeValueBlocksSize bool,
-) {
-	w.Printf("%9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7d %7.1f",
-		redact.Safe(m.NumFiles),
-		humanize.IEC.Int64(m.Size),
-		score,
-		humanize.IEC.Uint64(m.BytesIn),
-		humanize.IEC.Uint64(m.BytesIngested),
-		humanize.SI.Uint64(m.TablesIngested),
-		humanize.IEC.Uint64(m.BytesMoved),
-		humanize.SI.Uint64(m.TablesMoved),
-		humanize.IEC.Uint64(m.BytesFlushed+m.BytesCompacted),
-		humanize.SI.Uint64(m.TablesFlushed+m.TablesCompacted),
-		humanize.IEC.Uint64(m.BytesRead),
-		redact.Safe(m.Sublevels),
-		redact.Safe(m.WriteAmp()))
-	if includeValueBlocksSize {
-		w.Printf(" %7s\n", humanize.IEC.Uint64(m.Additional.ValueBlocksSize))
-	} else {
-		w.SafeString("\n")
-	}
-}
-
 // Metrics holds metrics for various subsystems of the DB such as the Cache,
 // Compactions, WAL, and per-Level metrics.
 //
@@ -158,14 +152,15 @@ type Metrics struct {
 
 	Compact struct {
 		// The total number of compactions, and per-compaction type counts.
-		Count            int64
+		Count             int64
-		DefaultCount     int64
+		DefaultCount      int64
-		DeleteOnlyCount  int64
+		DeleteOnlyCount   int64
-		ElisionOnlyCount int64
+		ElisionOnlyCount  int64
-		MoveCount        int64
+		MoveCount         int64
-		ReadCount        int64
+		ReadCount         int64
-		RewriteCount     int64
+		RewriteCount      int64
-		MultiLevelCount  int64
+		MultiLevelCount   int64
+		CounterLevelCount int64
 		// An estimate of the number of bytes that need to be compacted for the LSM
 		// to reach a stable state.
 		EstimatedDebt uint64
@@ -179,6 +174,14 @@ type Metrics struct {
 		// compaction. Such files are compacted in a rewrite compaction
 		// when no other compactions are picked.
 		MarkedFiles int
+		// Duration records the cumulative duration of all compactions since the
+		// database was opened.
+		Duration time.Duration
+	}
+
+	Ingest struct {
+		// The total number of ingestions
+		Count uint64
 	}
 
 	Flush struct {
@@ -210,7 +213,10 @@ type Metrics struct {
 		// The count of memtables.
 		Count int64
 		// The number of bytes present in zombie memtables which are no longer
-		// referenced by the current DB state but are still in use by an iterator.
+		// referenced by the current DB state. An unbounded number of memtables
+		// may be zombie if they're still in use by an iterator. One additional
+		// memtable may be zombie if it's no longer in use and waiting to be
+		// recycled.
 		ZombieSize uint64
 		// The count of zombie memtables.
 		ZombieCount int64
@@ -222,6 +228,9 @@ type Metrics struct {
 		// The approximate count of internal tombstones (DEL, SINGLEDEL and
 		// RANGEDEL key kinds) within the database.
 		TombstoneCount uint64
+		// A cumulative total number of missized DELSIZED keys encountered by
+		// compactions since the database was opened.
+		MissizedTombstonesCount uint64
 	}
 
 	Snapshots struct {
@@ -250,12 +259,18 @@ type Metrics struct {
 		ZombieSize uint64
 		// The count of zombie tables.
 		ZombieCount int64
+		// The count of the backing sstables.
+		BackingTableCount uint64
+		// The sum of the sizes of the all of the backing sstables.
+		BackingTableSize uint64
 	}
 
 	TableCache CacheMetrics
 
 	// Count of the number of open sstable iterators.
 	TableIters int64
+	// Uptime is the total time since this DB was opened.
+	Uptime time.Duration
 
 	WAL struct {
 		// Number of live WAL files.
@@ -281,6 +296,8 @@ type Metrics struct {
 		record.LogWriterMetrics
 	}
 
+	SecondaryCacheMetrics SecondaryCacheMetrics
+
 	private struct {
 		optionsFileSize  uint64
 		manifestFileSize uint64
@@ -294,6 +311,13 @@ var (
 		prometheus.LinearBuckets(0.0, float64(time.Microsecond*100), 50),
 		prometheus.ExponentialBucketsRange(float64(time.Millisecond*5), float64(10*time.Second), 50)...,
 	)
+
+	// SecondaryCacheIOBuckets exported to enable exporting from package pebble to
+	// enable exporting metrics with below buckets in CRDB.
+	SecondaryCacheIOBuckets = sharedcache.IOBuckets
+	// SecondaryCacheChannelWriteBuckets exported to enable exporting from package
+	// pebble to enable exporting metrics with below buckets in CRDB.
+	SecondaryCacheChannelWriteBuckets = sharedcache.ChannelWriteBuckets
 )
 
 // DiskSpaceUsage returns the total disk space used by the database in bytes,
@@ -313,12 +337,25 @@ func (m *Metrics) DiskSpaceUsage() uint64 {
 	return usageBytes
 }
 
-func (m *Metrics) levelSizes() [numLevels]int64 {
+// NumVirtual is the number of virtual sstables in the latest version
-	var sizes [numLevels]int64
+// summed over every level in the lsm.
-	for i := 0; i < len(sizes); i++ {
+func (m *Metrics) NumVirtual() uint64 {
-		sizes[i] = m.Levels[i].Size
+	var n uint64
+	for _, level := range m.Levels {
+		n += level.NumVirtualFiles
 	}
-	return sizes
+	return n
+}
+
+// VirtualSize is the sum of the sizes of the virtual sstables in the
+// latest version. BackingTableSize - VirtualSize gives an estimate for
+// the space amplification caused by not compacting virtual sstables.
+func (m *Metrics) VirtualSize() uint64 {
+	var size uint64
+	for _, level := range m.Levels {
+		size += level.VirtualSize
+	}
+	return size
 }
 
 // ReadAmp returns the current read amplification of the database.
@@ -349,59 +386,34 @@ func (m *Metrics) Total() LevelMetrics {
 	return total
 }
 
-const notApplicable = redact.SafeString("-")
+// String pretty-prints the metrics as below:
-
-func (m *Metrics) formatWAL(w redact.SafePrinter) {
-	var writeAmp float64
-	if m.WAL.BytesIn > 0 {
-		writeAmp = float64(m.WAL.BytesWritten) / float64(m.WAL.BytesIn)
-	}
-	w.Printf("    WAL %9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7.1f\n",
-		redact.Safe(m.WAL.Files),
-		humanize.Uint64(m.WAL.Size),
-		notApplicable,
-		humanize.Uint64(m.WAL.BytesIn),
-		notApplicable,
-		notApplicable,
-		notApplicable,
-		notApplicable,
-		humanize.Uint64(m.WAL.BytesWritten),
-		notApplicable,
-		notApplicable,
-		notApplicable,
-		redact.Safe(writeAmp))
-}
-
-// String pretty-prints the metrics, showing a line for the WAL, a line per-level, and
-// a total:
 //
-//	  __level_____count____size___score______in__ingest(sz_cnt)____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp
+//	      |                             |       |       |   ingested   |     moved    |    written   |       |    amp
-//	    WAL         1    28 B       -    17 B       -       -       -       -    56 B       -       -       -     3.3
+//	level | tables  size val-bl vtables | score |   in  | tables  size | tables  size | tables  size |  read |   r   w
-//	      0         1   770 B    0.25    28 B     0 B       0     0 B       0   770 B       1     0 B       1    27.5
+//	------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+---------
-//	      1         0     0 B    0.00     0 B     0 B       0     0 B       0     0 B       0     0 B       0     0.0
+//	    0 |   101   102B     0B       0 | 103.0 |  104B |   112   104B |   113   106B |   221   217B |  107B |   1  2.1
-//	      2         0     0 B    0.00     0 B     0 B       0     0 B       0     0 B       0     0 B       0     0.0
+//	    1 |   201   202B     0B       0 | 203.0 |  204B |   212   204B |   213   206B |   421   417B |  207B |   2  2.0
-//	      3         0     0 B    0.00     0 B     0 B       0     0 B       0     0 B       0     0 B       0     0.0
+//	    2 |   301   302B     0B       0 | 303.0 |  304B |   312   304B |   313   306B |   621   617B |  307B |   3  2.0
-//	      4         0     0 B    0.00     0 B     0 B       0     0 B       0     0 B       0     0 B       0     0.0
+//	    3 |   401   402B     0B       0 | 403.0 |  404B |   412   404B |   413   406B |   821   817B |  407B |   4  2.0
-//	      5         0     0 B    0.00     0 B     0 B       0     0 B       0     0 B       0     0 B       0     0.0
+//	    4 |   501   502B     0B       0 | 503.0 |  504B |   512   504B |   513   506B |  1.0K  1017B |  507B |   5  2.0
-//	      6         0     0 B       -     0 B     0 B       0     0 B       0     0 B       0     0 B       0     0.0
+//	    5 |   601   602B     0B       0 | 603.0 |  604B |   612   604B |   613   606B |  1.2K  1.2KB |  607B |   6  2.0
-//	  total         1   770 B       -    56 B     0 B       0     0 B       0   826 B       1     0 B       1    14.8
+//	    6 |   701   702B     0B       0 |     - |  704B |   712   704B |   713   706B |  1.4K  1.4KB |  707B |   7  2.0
-//	  flush         1                             0 B       0       0  (ingest = ingested-as-flushable, move = tables-ingested)
+//	total |  2.8K  2.7KB     0B       0 |     - | 2.8KB |  2.9K  2.8KB |  2.9K  2.8KB |  5.7K  8.4KB | 2.8KB |  28  3.0
-//	compact         0     0 B     0 B       0                          (size == estimated-debt, score = in-progress-bytes, in = num-in-progress)
+//	-------------------------------------------------------------------------------------------------------------------
-//	  ctype         0       0       0       0       0       0       0  (default, delete, elision, move, read, rewrite, multi-level)
+//	WAL: 22 files (24B)  in: 25B  written: 26B (4% overhead)
-//	 memtbl         1   256 K
+//	Flushes: 8
-//	zmemtbl         1   256 K
+//	Compactions: 5  estimated debt: 6B  in progress: 2 (7B)
-//	   ztbl         0     0 B
+//	default: 27  delete: 28  elision: 29  move: 30  read: 31  rewrite: 32  multi-level: 33
-//	 bcache         4   697 B    0.0%  (score == hit-rate)
+//	MemTables: 12 (11B)  zombie: 14 (13B)
-//	 tcache         1   696 B    0.0%  (score == hit-rate)
+//	Zombie tables: 16 (15B)
-//	  snaps         0       -       0  (score == earliest seq num)
+//	Backing tables: 0 (0B)
-//	 titers         1
+//	Block cache: 2 entries (1B)  hit rate: 42.9%
-//	 filter         -       -    0.0%  (score == utility)
+//	Table cache: 18 entries (17B)  hit rate: 48.7%
-//
+//	Secondary cache: 40 entries (40B)  hit rate: 49.9%
-// The WAL "in" metric is the size of the batches written to the WAL. The WAL
+//	Snapshots: 4  earliest seq num: 1024
-// "write" metric is the size of the physical data written to the WAL which
+//	Table iters: 21
-// includes record fragment overhead. Write amplification is computed as
+//	Filter utility: 47.4%
-// bytes-written / bytes-in, except for the total row where bytes-in is
+//	Ingestions: 27  as flushable: 36 (34B in 35 tables)
-// replaced with WAL-bytes-written + bytes-ingested.
 func (m *Metrics) String() string {
 	return redact.StringWithoutMarkers(m)
 }
@@ -422,29 +434,84 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 	// width specifiers. When the issue is fixed, we can convert these to
 	// RedactableStrings. https://github.com/cockroachdb/redact/issues/17
 
-	haveValueBlocks := false
+	multiExists := m.Compact.MultiLevelCount > 0
-	var valueBlocksHeading redact.SafeString
+	appendIfMulti := func(line redact.SafeString) {
-	for level := 0; level < numLevels; level++ {
+		if multiExists {
-		if m.Levels[level].Additional.ValueBlocksSize > 0 {
+			w.SafeString(line)
-			haveValueBlocks = true
-			valueBlocksHeading = "__val-bl"
-			break
 		}
 	}
+	newline := func() {
+		w.SafeString("\n")
+	}
+
+	w.SafeString("      |                             |       |       |   ingested   |     moved    |    written   |       |    amp")
+	appendIfMulti("   |     multilevel")
+	newline()
+	w.SafeString("level | tables  size val-bl vtables | score |   in  | tables  size | tables  size | tables  size |  read |   r   w")
+	appendIfMulti("  |    top   in  read")
+	newline()
+	w.SafeString("------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+---------")
+	appendIfMulti("-+------------------")
+	newline()
+
+	// formatRow prints out a row of the table.
+	formatRow := func(m *LevelMetrics, score float64) {
+		scoreStr := "-"
+		if !math.IsNaN(score) {
+			// Try to keep the string no longer than 5 characters.
+			switch {
+			case score < 99.995:
+				scoreStr = fmt.Sprintf("%.2f", score)
+			case score < 999.95:
+				scoreStr = fmt.Sprintf("%.1f", score)
+			default:
+				scoreStr = fmt.Sprintf("%.0f", score)
+			}
+		}
+		var wampStr string
+		if wamp := m.WriteAmp(); wamp > 99.5 {
+			wampStr = fmt.Sprintf("%.0f", wamp)
+		} else {
+			wampStr = fmt.Sprintf("%.1f", wamp)
+		}
+
+		w.Printf("| %5s %6s %6s %7s | %5s | %5s | %5s %6s | %5s %6s | %5s %6s | %5s | %3d %4s",
+			humanize.Count.Int64(m.NumFiles),
+			humanize.Bytes.Int64(m.Size),
+			humanize.Bytes.Uint64(m.Additional.ValueBlocksSize),
+			humanize.Count.Uint64(m.NumVirtualFiles),
+			redact.Safe(scoreStr),
+			humanize.Bytes.Uint64(m.BytesIn),
+			humanize.Count.Uint64(m.TablesIngested),
+			humanize.Bytes.Uint64(m.BytesIngested),
+			humanize.Count.Uint64(m.TablesMoved),
+			humanize.Bytes.Uint64(m.BytesMoved),
+			humanize.Count.Uint64(m.TablesFlushed+m.TablesCompacted),
+			humanize.Bytes.Uint64(m.BytesFlushed+m.BytesCompacted),
+			humanize.Bytes.Uint64(m.BytesRead),
+			redact.Safe(m.Sublevels),
+			redact.Safe(wampStr))
+
+		if multiExists {
+			w.Printf(" | %5s %5s %5s",
+				humanize.Bytes.Uint64(m.MultiLevel.BytesInTop),
+				humanize.Bytes.Uint64(m.MultiLevel.BytesIn),
+				humanize.Bytes.Uint64(m.MultiLevel.BytesRead))
+		}
+		newline()
+	}
+
 	var total LevelMetrics
-	w.Printf("__level_____count____size___score______in__ingest(sz_cnt)"+
-		"____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp%s\n", valueBlocksHeading)
-	m.formatWAL(w)
 	for level := 0; level < numLevels; level++ {
 		l := &m.Levels[level]
-		w.Printf("%7d ", redact.Safe(level))
+		w.Printf("%5d ", redact.Safe(level))
 
 		// Format the score.
-		var score redact.SafeValue = notApplicable
+		score := math.NaN()
 		if level < numLevels-1 {
-			score = redact.Safe(fmt.Sprintf("%0.2f", l.Score))
+			score = l.Score
 		}
-		l.format(w, score, haveValueBlocks)
+		formatRow(l, score)
 		total.Add(l)
 		total.Sublevels += l.Sublevels
 	}
@@ -454,57 +521,105 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 	// the bytes written to the log and bytes written externally and then
 	// ingested.
 	total.BytesFlushed += total.BytesIn
-	w.SafeString("  total ")
+	w.SafeString("total ")
-	total.format(w, notApplicable, haveValueBlocks)
+	formatRow(&total, math.NaN())
 
-	w.Printf("  flush %9d %31s %7d %7d  %s\n",
+	w.SafeString("-------------------------------------------------------------------------------------------------------------------")
-		redact.Safe(m.Flush.Count),
+	appendIfMulti("--------------------")
-		humanize.IEC.Uint64(m.Flush.AsIngestBytes),
+	newline()
-		redact.Safe(m.Flush.AsIngestTableCount),
+	w.Printf("WAL: %d files (%s)  in: %s  written: %s (%.0f%% overhead)\n",
-		redact.Safe(m.Flush.AsIngestCount),
+		redact.Safe(m.WAL.Files),
-		redact.SafeString(`(ingest = tables-ingested, move = ingested-as-flushable)`))
+		humanize.Bytes.Uint64(m.WAL.Size),
-	w.Printf("compact %9d %7s %7s %7d %s %s\n",
+		humanize.Bytes.Uint64(m.WAL.BytesIn),
+		humanize.Bytes.Uint64(m.WAL.BytesWritten),
+		redact.Safe(percent(int64(m.WAL.BytesWritten)-int64(m.WAL.BytesIn), int64(m.WAL.BytesIn))))
+
+	w.Printf("Flushes: %d\n", redact.Safe(m.Flush.Count))
+
+	w.Printf("Compactions: %d  estimated debt: %s  in progress: %d (%s)\n",
 		redact.Safe(m.Compact.Count),
-		humanize.IEC.Uint64(m.Compact.EstimatedDebt),
+		humanize.Bytes.Uint64(m.Compact.EstimatedDebt),
-		humanize.IEC.Int64(m.Compact.InProgressBytes),
 		redact.Safe(m.Compact.NumInProgress),
-		redact.SafeString(strings.Repeat(" ", 24)),
+		humanize.Bytes.Int64(m.Compact.InProgressBytes))
-		redact.SafeString(`(size == estimated-debt, score = in-progress-bytes, in = num-in-progress)`))
+
-	w.Printf("  ctype %9d %7d %7d %7d %7d %7d %7d  %s\n",
+	w.Printf("             default: %d  delete: %d  elision: %d  move: %d  read: %d  rewrite: %d  multi-level: %d\n",
 		redact.Safe(m.Compact.DefaultCount),
 		redact.Safe(m.Compact.DeleteOnlyCount),
 		redact.Safe(m.Compact.ElisionOnlyCount),
 		redact.Safe(m.Compact.MoveCount),
 		redact.Safe(m.Compact.ReadCount),
 		redact.Safe(m.Compact.RewriteCount),
-		redact.Safe(m.Compact.MultiLevelCount),
+		redact.Safe(m.Compact.MultiLevelCount))
-		redact.SafeString(`(default, delete, elision, move, read, rewrite, multi-level)`))
+
-	w.Printf(" memtbl %9d %7s\n",
+	w.Printf("MemTables: %d (%s)  zombie: %d (%s)\n",
 		redact.Safe(m.MemTable.Count),
-		humanize.IEC.Uint64(m.MemTable.Size))
+		humanize.Bytes.Uint64(m.MemTable.Size),
-	w.Printf("zmemtbl %9d %7s\n",
 		redact.Safe(m.MemTable.ZombieCount),
-		humanize.IEC.Uint64(m.MemTable.ZombieSize))
+		humanize.Bytes.Uint64(m.MemTable.ZombieSize))
-	w.Printf("   ztbl %9d %7s\n",
+
+	w.Printf("Zombie tables: %d (%s)\n",
 		redact.Safe(m.Table.ZombieCount),
-		humanize.IEC.Uint64(m.Table.ZombieSize))
+		humanize.Bytes.Uint64(m.Table.ZombieSize))
-	formatCacheMetrics(w, &m.BlockCache, "bcache")
+
-	formatCacheMetrics(w, &m.TableCache, "tcache")
+	w.Printf("Backing tables: %d (%s)\n",
-	w.Printf("  snaps %9d %7s %7d  (score == earliest seq num)\n",
+		redact.Safe(m.Table.BackingTableCount),
+		humanize.Bytes.Uint64(m.Table.BackingTableSize))
+	w.Printf("Virtual tables: %d (%s)\n",
+		redact.Safe(m.NumVirtual()),
+		humanize.Bytes.Uint64(m.VirtualSize()))
+
+	formatCacheMetrics := func(m *CacheMetrics, name redact.SafeString) {
+		w.Printf("%s: %s entries (%s)  hit rate: %.1f%%\n",
+			name,
+			humanize.Count.Int64(m.Count),
+			humanize.Bytes.Int64(m.Size),
+			redact.Safe(hitRate(m.Hits, m.Misses)))
+	}
+	formatCacheMetrics(&m.BlockCache, "Block cache")
+	formatCacheMetrics(&m.TableCache, "Table cache")
+
+	formatSharedCacheMetrics := func(w redact.SafePrinter, m *SecondaryCacheMetrics, name redact.SafeString) {
+		w.Printf("%s: %s entries (%s)  hit rate: %.1f%%\n",
+			name,
+			humanize.Count.Int64(m.Count),
+			humanize.Bytes.Int64(m.Size),
+			redact.Safe(hitRate(m.ReadsWithFullHit, m.ReadsWithPartialHit+m.ReadsWithNoHit)))
+	}
+	formatSharedCacheMetrics(w, &m.SecondaryCacheMetrics, "Secondary cache")
+
+	w.Printf("Snapshots: %d  earliest seq num: %d\n",
 		redact.Safe(m.Snapshots.Count),
-		notApplicable,
 		redact.Safe(m.Snapshots.EarliestSeqNum))
-	w.Printf(" titers %9d\n", redact.Safe(m.TableIters))
+
-	w.Printf(" filter %9s %7s %6.1f%%  (score == utility)\n",
+	w.Printf("Table iters: %d\n", redact.Safe(m.TableIters))
-		notApplicable,
+	w.Printf("Filter utility: %.1f%%\n", redact.Safe(hitRate(m.Filter.Hits, m.Filter.Misses)))
-		notApplicable,
+	w.Printf("Ingestions: %d  as flushable: %d (%s in %d tables)\n",
-		redact.Safe(hitRate(m.Filter.Hits, m.Filter.Misses)))
+		redact.Safe(m.Ingest.Count),
+		redact.Safe(m.Flush.AsIngestCount),
+		humanize.Bytes.Uint64(m.Flush.AsIngestBytes),
+		redact.Safe(m.Flush.AsIngestTableCount))
 }
 
 func hitRate(hits, misses int64) float64 {
-	sum := hits + misses
+	return percent(hits, hits+misses)
-	if sum == 0 {
+}
+
+func percent(numerator, denominator int64) float64 {
+	if denominator == 0 {
 		return 0
 	}
-	return 100 * float64(hits) / float64(sum)
+	return 100 * float64(numerator) / float64(denominator)
+}
+
+// StringForTests is identical to m.String() on 64-bit platforms. It is used to
+// provide a platform-independent result for tests.
+func (m *Metrics) StringForTests() string {
+	mCopy := *m
+	if math.MaxInt == math.MaxInt32 {
+		// This is the difference in Sizeof(sstable.Reader{})) between 64 and 32 bit
+		// platforms.
+		const tableCacheSizeAdjustment = 212
+		mCopy.TableCache.Size += mCopy.TableCache.Count * tableCacheSizeAdjustment
+	}
+	return redact.StringWithoutMarkers(&mCopy)
 }

vendor/github.com/cockroachdb/pebble/objstorage/noop_readahead.go

@@ -4,26 +4,31 @@
 
 package objstorage
 
-import "io"
+import "context"
 
 // NoopReadHandle can be used by Readable implementations that don't
 // support read-ahead.
 type NoopReadHandle struct {
-	io.ReaderAt
+	readable Readable
 }
 
 // MakeNoopReadHandle initializes a NoopReadHandle.
-func MakeNoopReadHandle(r io.ReaderAt) NoopReadHandle {
+func MakeNoopReadHandle(r Readable) NoopReadHandle {
-	return NoopReadHandle{ReaderAt: r}
+	return NoopReadHandle{readable: r}
 }
 
 var _ ReadHandle = (*NoopReadHandle)(nil)
 
+// ReadAt is part of the ReadHandle interface.
+func (h *NoopReadHandle) ReadAt(ctx context.Context, p []byte, off int64) error {
+	return h.readable.ReadAt(ctx, p, off)
+}
+
 // Close is part of the ReadHandle interface.
 func (*NoopReadHandle) Close() error { return nil }
 
-// MaxReadahead is part of the ReadHandle interface.
+// SetupForCompaction is part of the ReadHandle interface.
-func (*NoopReadHandle) MaxReadahead() {}
+func (*NoopReadHandle) SetupForCompaction() {}
 
 // RecordCacheHit is part of the ReadHandle interface.
-func (*NoopReadHandle) RecordCacheHit(offset, size int64) {}
+func (*NoopReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {}

vendor/github.com/cockroachdb/pebble/objstorage/objstorage.go

@@ -0,0 +1,324 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorage
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// Readable is the handle for an object that is open for reading.
+type Readable interface {
+	// ReadAt reads len(p) bytes into p starting at offset off.
+	//
+	// Does not return partial results; if off + len(p) is past the end of the
+	// object, an error is returned.
+	//
+	// Clients of ReadAt can execute parallel ReadAt calls on the
+	// same Readable.
+	ReadAt(ctx context.Context, p []byte, off int64) error
+
+	Close() error
+
+	// Size returns the size of the object.
+	Size() int64
+
+	// NewReadHandle creates a read handle for ReadAt requests that are related
+	// and can benefit from optimizations like read-ahead.
+	//
+	// The ReadHandle must be closed before the Readable is closed.
+	//
+	// Multiple separate ReadHandles can be used.
+	NewReadHandle(ctx context.Context) ReadHandle
+}
+
+// ReadHandle is used to perform reads that are related and might benefit from
+// optimizations like read-ahead.
+type ReadHandle interface {
+	// ReadAt reads len(p) bytes into p starting at offset off.
+	//
+	// Does not return partial results; if off + len(p) is past the end of the
+	// object, an error is returned.
+	//
+	// Parallel ReadAt calls on the same ReadHandle are not allowed.
+	ReadAt(ctx context.Context, p []byte, off int64) error
+
+	Close() error
+
+	// SetupForCompaction informs the implementation that the read handle will
+	// be used to read data blocks for a compaction. The implementation can expect
+	// sequential reads, and can decide to not retain data in any caches.
+	SetupForCompaction()
+
+	// RecordCacheHit informs the implementation that we were able to retrieve a
+	// block from cache. This is useful for example when the implementation is
+	// trying to detect a sequential reading pattern.
+	RecordCacheHit(ctx context.Context, offset, size int64)
+}
+
+// Writable is the handle for an object that is open for writing.
+// Either Finish or Abort must be called.
+type Writable interface {
+	// Write writes len(p) bytes from p to the underlying object. The data is not
+	// guaranteed to be durable until Finish is called.
+	//
+	// Note that Write *is* allowed to modify the slice passed in, whether
+	// temporarily or permanently. Callers of Write need to take this into
+	// account.
+	Write(p []byte) error
+
+	// Finish completes the object and makes the data durable.
+	// No further calls are allowed after calling Finish.
+	Finish() error
+
+	// Abort gives up on finishing the object. There is no guarantee about whether
+	// the object exists after calling Abort.
+	// No further calls are allowed after calling Abort.
+	Abort()
+}
+
+// ObjectMetadata contains the metadata required to be able to access an object.
+type ObjectMetadata struct {
+	DiskFileNum base.DiskFileNum
+	FileType    base.FileType
+
+	// The fields below are only set if the object is on remote storage.
+	Remote struct {
+		// CreatorID identifies the DB instance that originally created the object.
+		//
+		// Only used when CustomObjectName is not set.
+		CreatorID CreatorID
+		// CreatorFileNum is the identifier for the object within the context of the
+		// DB instance that originally created the object.
+		//
+		// Only used when CustomObjectName is not set.
+		CreatorFileNum base.DiskFileNum
+		// CustomObjectName (if it is set) overrides the object name that is normally
+		// derived from the CreatorID and CreatorFileNum.
+		CustomObjectName string
+		// CleanupMethod indicates the method for cleaning up unused shared objects.
+		CleanupMethod SharedCleanupMethod
+		// Locator identifies the remote.Storage implementation for this object.
+		Locator remote.Locator
+		// Storage is the remote.Storage object corresponding to the Locator. Used
+		// to avoid lookups in hot paths.
+		Storage remote.Storage
+	}
+}
+
+// IsRemote returns true if the object is on remote storage.
+func (meta *ObjectMetadata) IsRemote() bool {
+	return meta.IsShared() || meta.IsExternal()
+}
+
+// IsExternal returns true if the object is on remote storage but is not owned
+// by any Pebble instances in the cluster.
+func (meta *ObjectMetadata) IsExternal() bool {
+	return meta.Remote.CustomObjectName != ""
+}
+
+// IsShared returns true if the object is on remote storage and is owned by a
+// Pebble instance in the cluster (potentially shared between multiple
+// instances).
+func (meta *ObjectMetadata) IsShared() bool {
+	return meta.Remote.CreatorID.IsSet()
+}
+
+// AssertValid checks that the metadata is sane.
+func (meta *ObjectMetadata) AssertValid() {
+	if !meta.IsRemote() {
+		// Verify all Remote fields are empty.
+		if meta.Remote != (ObjectMetadata{}).Remote {
+			panic(errors.AssertionFailedf("meta.Remote not empty: %#v", meta.Remote))
+		}
+	} else {
+		if meta.Remote.CustomObjectName != "" {
+			if meta.Remote.CreatorID == 0 {
+				panic(errors.AssertionFailedf("CreatorID not set"))
+			}
+			if meta.Remote.CreatorFileNum == base.FileNum(0).DiskFileNum() {
+				panic(errors.AssertionFailedf("CreatorFileNum not set"))
+			}
+		}
+		if meta.Remote.CleanupMethod != SharedNoCleanup && meta.Remote.CleanupMethod != SharedRefTracking {
+			panic(errors.AssertionFailedf("invalid CleanupMethod %d", meta.Remote.CleanupMethod))
+		}
+		if meta.Remote.Storage == nil {
+			panic(errors.AssertionFailedf("Storage not set"))
+		}
+	}
+}
+
+// CreatorID identifies the DB instance that originally created a shared object.
+// This ID is incorporated in backing object names.
+// Must be non-zero.
+type CreatorID uint64
+
+// IsSet returns true if the CreatorID is not zero.
+func (c CreatorID) IsSet() bool { return c != 0 }
+
+func (c CreatorID) String() string { return fmt.Sprintf("%d", c) }
+
+// SharedCleanupMethod indicates the method for cleaning up unused shared objects.
+type SharedCleanupMethod uint8
+
+const (
+	// SharedRefTracking is used for shared objects for which objstorage providers
+	// keep track of references via reference marker objects.
+	SharedRefTracking SharedCleanupMethod = iota
+
+	// SharedNoCleanup is used for remote objects that are managed externally; the
+	// objstorage provider never deletes such objects.
+	SharedNoCleanup
+)
+
+// OpenOptions contains optional arguments for OpenForReading.
+type OpenOptions struct {
+	// MustExist triggers a fatal error if the file does not exist. The fatal
+	// error message contains extra information helpful for debugging.
+	MustExist bool
+}
+
+// CreateOptions contains optional arguments for Create.
+type CreateOptions struct {
+	// PreferSharedStorage causes the object to be created on shared storage if
+	// the provider has shared storage configured.
+	PreferSharedStorage bool
+
+	// SharedCleanupMethod is used for the object when it is created on shared storage.
+	// The default (zero) value is SharedRefTracking.
+	SharedCleanupMethod SharedCleanupMethod
+}
+
+// Provider is a singleton object used to access and manage objects.
+//
+// An object is conceptually like a large immutable file. The main use of
+// objects is for storing sstables; in the future it could also be used for blob
+// storage.
+//
+// The Provider can only manage objects that it knows about - either objects
+// created by the provider, or existing objects the Provider was informed about
+// via AddObjects.
+//
+// Objects are currently backed by a vfs.File or a remote.Storage object.
+type Provider interface {
+	// OpenForReading opens an existing object.
+	OpenForReading(
+		ctx context.Context, fileType base.FileType, FileNum base.DiskFileNum, opts OpenOptions,
+	) (Readable, error)
+
+	// Create creates a new object and opens it for writing.
+	//
+	// The object is not guaranteed to be durable (accessible in case of crashes)
+	// until Sync is called.
+	Create(
+		ctx context.Context, fileType base.FileType, FileNum base.DiskFileNum, opts CreateOptions,
+	) (w Writable, meta ObjectMetadata, err error)
+
+	// Remove removes an object.
+	//
+	// The object is not guaranteed to be durably removed until Sync is called.
+	Remove(fileType base.FileType, FileNum base.DiskFileNum) error
+
+	// Sync flushes the metadata from creation or removal of objects since the last Sync.
+	// This includes objects that have been Created but for which
+	// Writable.Finish() has not yet been called.
+	Sync() error
+
+	// LinkOrCopyFromLocal creates a new object that is either a copy of a given
+	// local file or a hard link (if the new object is created on the same FS, and
+	// if the FS supports it).
+	//
+	// The object is not guaranteed to be durable (accessible in case of crashes)
+	// until Sync is called.
+	LinkOrCopyFromLocal(
+		ctx context.Context,
+		srcFS vfs.FS,
+		srcFilePath string,
+		dstFileType base.FileType,
+		dstFileNum base.DiskFileNum,
+		opts CreateOptions,
+	) (ObjectMetadata, error)
+
+	// Lookup returns the metadata of an object that is already known to the Provider.
+	// Does not perform any I/O.
+	Lookup(fileType base.FileType, FileNum base.DiskFileNum) (ObjectMetadata, error)
+
+	// Path returns an internal, implementation-dependent path for the object. It is
+	// meant to be used for informational purposes (like logging).
+	Path(meta ObjectMetadata) string
+
+	// Size returns the size of the object.
+	Size(meta ObjectMetadata) (int64, error)
+
+	// List returns the objects currently known to the provider. Does not perform any I/O.
+	List() []ObjectMetadata
+
+	// SetCreatorID sets the CreatorID which is needed in order to use shared
+	// objects. Remote object usage is disabled until this method is called the
+	// first time. Once set, the Creator ID is persisted and cannot change.
+	//
+	// Cannot be called if shared storage is not configured for the provider.
+	SetCreatorID(creatorID CreatorID) error
+
+	// IsSharedForeign returns whether this object is owned by a different node.
+	IsSharedForeign(meta ObjectMetadata) bool
+
+	// RemoteObjectBacking encodes the remote object metadata for the given object.
+	RemoteObjectBacking(meta *ObjectMetadata) (RemoteObjectBackingHandle, error)
+
+	// CreateExternalObjectBacking creates a backing for an existing object with a
+	// custom object name. The object is considered to be managed outside of
+	// Pebble and will never be removed by Pebble.
+	CreateExternalObjectBacking(locator remote.Locator, objName string) (RemoteObjectBacking, error)
+
+	// AttachRemoteObjects registers existing remote objects with this provider.
+	AttachRemoteObjects(objs []RemoteObjectToAttach) ([]ObjectMetadata, error)
+
+	Close() error
+
+	// IsNotExistError indicates whether the error is known to report that a file or
+	// directory does not exist.
+	IsNotExistError(err error) bool
+
+	// Metrics returns metrics about objstorage. Currently, it only returns metrics
+	// about the shared cache.
+	Metrics() sharedcache.Metrics
+}
+
+// RemoteObjectBacking encodes the metadata necessary to incorporate a shared
+// object into a different Pebble instance. The encoding is specific to a given
+// Provider implementation.
+type RemoteObjectBacking []byte
+
+// RemoteObjectBackingHandle is a container for a RemoteObjectBacking which
+// ensures that the backing stays valid. A backing can otherwise become invalid
+// if this provider unrefs the shared object. The RemoteObjectBackingHandle
+// delays any unref until Close.
+type RemoteObjectBackingHandle interface {
+	// Get returns the backing. The backing is only guaranteed to be valid until
+	// Close is called (or until the Provider is closed). If Close was already
+	// called, returns an error.
+	Get() (RemoteObjectBacking, error)
+	Close()
+}
+
+// RemoteObjectToAttach contains the arguments needed to attach an existing remote object.
+type RemoteObjectToAttach struct {
+	// FileNum is the file number that will be used to refer to this object (in
+	// the context of this instance).
+	FileNum  base.DiskFileNum
+	FileType base.FileType
+	// Backing contains the metadata for the remote object backing (normally
+	// generated from a different instance, but using the same Provider
+	// implementation).
+	Backing RemoteObjectBacking
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing.go

@@ -0,0 +1,68 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objiotracing
+
+import "github.com/cockroachdb/pebble/internal/base"
+
+// OpType indicates the type of operation.
+type OpType uint8
+
+// OpType values.
+const (
+	ReadOp OpType = iota
+	WriteOp
+	// RecordCacheHitOp happens when a read is satisfied from the block cache. See
+	// objstorage.ReadHandle.RecordCacheHit().
+	RecordCacheHitOp
+	// SetupForCompactionOp is a "meta operation" that configures a read handle
+	// for large sequential reads. See objstorage.ReadHandle.SetupForCompaction().
+	SetupForCompactionOp
+)
+
+// Reason indicates the higher-level context of the operation.
+type Reason uint8
+
+// Reason values.
+const (
+	UnknownReason Reason = iota
+	ForFlush
+	ForCompaction
+	ForIngestion
+	// TODO(radu): add ForUserFacing.
+)
+
+// BlockType indicates the type of data block relevant to an operation.
+type BlockType uint8
+
+// BlockType values.
+const (
+	UnknownBlock BlockType = iota
+	DataBlock
+	ValueBlock
+	FilterBlock
+	MetadataBlock
+)
+
+// Event is the on-disk format of a tracing event. It is exported here so that
+// trace processing tools can use it by importing this package.
+type Event struct {
+	// Event start time as a Unix time (see time.Time.StartUnixNano()).
+	// Note that recorded events are not necessarily ordered by time - this is
+	// because separate event "streams" use local buffers (for performance).
+	StartUnixNano int64
+	Op            OpType
+	Reason        Reason
+	BlockType     BlockType
+	// LSM level plus one (with 0 indicating unknown level).
+	LevelPlusOne uint8
+	// Hardcoded padding so that struct layout doesn't depend on architecture.
+	_       uint32
+	FileNum base.FileNum
+	// HandleID is a unique identifier corresponding to an objstorage.ReadHandle;
+	// only set for read operations performed through a ReadHandle.
+	HandleID uint64
+	Offset   int64
+	Size     int64
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go

@@ -0,0 +1,59 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build !pebble_obj_io_tracing
+// +build !pebble_obj_io_tracing
+
+package objiotracing
+
+import (
+	"context"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// Enabled is used to short circuit tracing-related code in regular builds.
+const Enabled = false
+
+// Tracer manages the writing of object IO traces to files.
+type Tracer struct{}
+
+// Open creates a Tracer which generates trace files in the given directory.
+// Each trace file contains a series of Events (as they are in memory).
+func Open(fs vfs.FS, fsDir string) *Tracer {
+	return nil
+}
+
+// Close the tracer, flushing any remaining events.
+func (*Tracer) Close() {}
+
+// WrapReadable wraps an objstorage.Readable with one that generates tracing
+// events.
+func (*Tracer) WrapReadable(
+	ctx context.Context, r objstorage.Readable, fileNum base.DiskFileNum,
+) objstorage.Readable {
+	return r
+}
+
+// WrapWritable wraps an objstorage.Writable with one that generates tracing
+// events.
+func (t *Tracer) WrapWritable(
+	ctx context.Context, w objstorage.Writable, fileNum base.DiskFileNum,
+) objstorage.Writable {
+	return w
+}
+
+// WithReason creates a context that has an associated Reason (which ends up in
+// traces created under that context).
+func WithReason(ctx context.Context, reason Reason) context.Context { return ctx }
+
+// WithBlockType creates a context that has an associated BlockType (which ends up in
+// traces created under that context).
+func WithBlockType(ctx context.Context, blockType BlockType) context.Context { return ctx }
+
+// WithLevel creates a context that has an associated level (which ends up in
+// traces created under that context).
+func WithLevel(ctx context.Context, level int) context.Context { return ctx }

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go

@@ -0,0 +1,410 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+//go:build pebble_obj_io_tracing
+// +build pebble_obj_io_tracing
+
+package objiotracing
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+	"time"
+	"unsafe"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// Enabled is used to short circuit tracing-related code in regular builds.
+const Enabled = true
+
+// Tracer manages the writing of object IO traces to files.
+//
+// The tracer runs a background worker goroutine which receives trace event
+// buffers over a channel and dumps them to IOTRACES- files. Wrapper
+// implementations for Readable, ReadHandle, Writable are producers of traces;
+// they maintain internal buffers of events which get flushed to the buffered
+// channel when they get full. This allows for minimal synchronization per IO
+// (as for most of these structures, an instance only allows a single IO at a
+// time).
+type Tracer struct {
+	fs    vfs.FS
+	fsDir string
+
+	handleID atomic.Uint64
+
+	workerStopCh chan struct{}
+	workerDataCh chan eventBuf
+	workerWait   sync.WaitGroup
+}
+
+// Open creates a Tracer which generates trace files in the given directory.
+// Each trace file contains a series of Events (as they are in memory).
+func Open(fs vfs.FS, fsDir string) *Tracer {
+	t := &Tracer{
+		fs:           fs,
+		fsDir:        fsDir,
+		workerStopCh: make(chan struct{}),
+		workerDataCh: make(chan eventBuf, channelBufSize),
+	}
+
+	t.handleID.Store(uint64(rand.NewSource(time.Now().UnixNano()).Int63()))
+
+	t.workerWait.Add(1)
+	go t.workerLoop()
+	return t
+}
+
+// Close the tracer, flushing any remaining events.
+func (t *Tracer) Close() {
+	if t.workerStopCh == nil {
+		return
+	}
+	// Tell the worker to stop and wait for it to finish up.
+	close(t.workerStopCh)
+	t.workerWait.Wait()
+	t.workerStopCh = nil
+}
+
+// WrapWritable wraps an objstorage.Writable with one that generates tracing
+// events.
+func (t *Tracer) WrapWritable(
+	ctx context.Context, w objstorage.Writable, fileNum base.FileNum,
+) objstorage.Writable {
+	return &writable{
+		w:       w,
+		fileNum: fileNum,
+		g:       makeEventGenerator(ctx, t),
+	}
+}
+
+type writable struct {
+	w         objstorage.Writable
+	fileNum   base.FileNum
+	curOffset int64
+	g         eventGenerator
+}
+
+var _ objstorage.Writable = (*writable)(nil)
+
+// Write is part of the objstorage.Writable interface.
+func (w *writable) Write(p []byte) error {
+	w.g.add(context.Background(), Event{
+		Op:      WriteOp,
+		FileNum: w.fileNum,
+		Offset:  w.curOffset,
+		Size:    int64(len(p)),
+	})
+	// If w.w.Write(p) returns an error, a new writable
+	// will be used, so even tho all of p may not have
+	// been written to the underlying "file", it is okay
+	// to add len(p) to curOffset.
+	w.curOffset += int64(len(p))
+	return w.w.Write(p)
+}
+
+// Finish is part of the objstorage.Writable interface.
+func (w *writable) Finish() error {
+	w.g.flush()
+	return w.w.Finish()
+}
+
+// Abort is part of the objstorage.Writable interface.
+func (w *writable) Abort() {
+	w.g.flush()
+	w.w.Abort()
+}
+
+// WrapReadable wraps an objstorage.Readable with one that generates tracing
+// events.
+func (t *Tracer) WrapReadable(
+	ctx context.Context, r objstorage.Readable, fileNum base.FileNum,
+) objstorage.Readable {
+	res := &readable{
+		r:       r,
+		fileNum: fileNum,
+	}
+	res.mu.g = makeEventGenerator(ctx, t)
+	return res
+}
+
+type readable struct {
+	r       objstorage.Readable
+	fileNum base.FileNum
+	mu      struct {
+		sync.Mutex
+		g eventGenerator
+	}
+}
+
+var _ objstorage.Readable = (*readable)(nil)
+
+// ReadAt is part of the objstorage.Readable interface.
+func (r *readable) ReadAt(ctx context.Context, v []byte, off int64) (n int, err error) {
+	r.mu.Lock()
+	r.mu.g.add(ctx, Event{
+		Op:      ReadOp,
+		FileNum: r.fileNum,
+		Offset:  off,
+		Size:    int64(len(v)),
+	})
+	r.mu.Unlock()
+	return r.r.ReadAt(ctx, v, off)
+}
+
+// Close is part of the objstorage.Readable interface.
+func (r *readable) Close() error {
+	r.mu.g.flush()
+	return r.r.Close()
+}
+
+// Size is part of the objstorage.Readable interface.
+func (r *readable) Size() int64 {
+	return r.r.Size()
+}
+
+// NewReadHandle is part of the objstorage.Readable interface.
+func (r *readable) NewReadHandle(ctx context.Context) objstorage.ReadHandle {
+	// It's safe to get the tracer from the generator without the mutex since it never changes.
+	t := r.mu.g.t
+	return &readHandle{
+		rh:       r.r.NewReadHandle(ctx),
+		fileNum:  r.fileNum,
+		handleID: t.handleID.Add(1),
+		g:        makeEventGenerator(ctx, t),
+	}
+}
+
+type readHandle struct {
+	rh       objstorage.ReadHandle
+	fileNum  base.FileNum
+	handleID uint64
+	g        eventGenerator
+}
+
+var _ objstorage.ReadHandle = (*readHandle)(nil)
+
+// ReadAt is part of the objstorage.ReadHandle interface.
+func (rh *readHandle) ReadAt(ctx context.Context, p []byte, off int64) (n int, err error) {
+	rh.g.add(ctx, Event{
+		Op:       ReadOp,
+		FileNum:  rh.fileNum,
+		HandleID: rh.handleID,
+		Offset:   off,
+		Size:     int64(len(p)),
+	})
+	return rh.rh.ReadAt(ctx, p, off)
+}
+
+// Close is part of the objstorage.ReadHandle interface.
+func (rh *readHandle) Close() error {
+	rh.g.flush()
+	return rh.rh.Close()
+}
+
+// SetupForCompaction is part of the objstorage.ReadHandle interface.
+func (rh *readHandle) SetupForCompaction() {
+	rh.g.add(context.Background(), Event{
+		Op:       SetupForCompactionOp,
+		FileNum:  rh.fileNum,
+		HandleID: rh.handleID,
+	})
+	rh.rh.SetupForCompaction()
+}
+
+// RecordCacheHit is part of the objstorage.ReadHandle interface.
+func (rh *readHandle) RecordCacheHit(ctx context.Context, offset, size int64) {
+	rh.g.add(ctx, Event{
+		Op:       RecordCacheHitOp,
+		FileNum:  rh.fileNum,
+		HandleID: rh.handleID,
+		Offset:   offset,
+		Size:     size,
+	})
+	rh.rh.RecordCacheHit(ctx, offset, size)
+}
+
+type ctxInfo struct {
+	reason       Reason
+	blockType    BlockType
+	levelPlusOne uint8
+}
+
+func mergeCtxInfo(base, other ctxInfo) ctxInfo {
+	res := other
+	if res.reason == 0 {
+		res.reason = base.reason
+	}
+	if res.blockType == 0 {
+		res.blockType = base.blockType
+	}
+	if res.levelPlusOne == 0 {
+		res.levelPlusOne = base.levelPlusOne
+	}
+	return res
+}
+
+type ctxInfoKey struct{}
+
+func withInfo(ctx context.Context, info ctxInfo) context.Context {
+	return context.WithValue(ctx, ctxInfoKey{}, info)
+}
+
+func infoFromCtx(ctx context.Context) ctxInfo {
+	res := ctx.Value(ctxInfoKey{})
+	if res == nil {
+		return ctxInfo{}
+	}
+	return res.(ctxInfo)
+}
+
+// WithReason creates a context that has an associated Reason (which ends up in
+// traces created under that context).
+func WithReason(ctx context.Context, reason Reason) context.Context {
+	info := infoFromCtx(ctx)
+	info.reason = reason
+	return withInfo(ctx, info)
+}
+
+// WithBlockType creates a context that has an associated BlockType (which ends up in
+// traces created under that context).
+func WithBlockType(ctx context.Context, blockType BlockType) context.Context {
+	info := infoFromCtx(ctx)
+	info.blockType = blockType
+	return withInfo(ctx, info)
+}
+
+// WithLevel creates a context that has an associated level (which ends up in
+// traces created under that context).
+func WithLevel(ctx context.Context, level int) context.Context {
+	info := infoFromCtx(ctx)
+	info.levelPlusOne = uint8(level) + 1
+	return withInfo(ctx, info)
+}
+
+const (
+	eventSize            = int(unsafe.Sizeof(Event{}))
+	targetEntriesPerFile = 256 * 1024 * 1024 / eventSize // 256MB files
+	eventsPerBuf         = 16
+	channelBufSize       = 512 * 1024 / eventsPerBuf // 512K events.
+	bytesPerFileSync     = 128 * 1024
+)
+
+type eventBuf struct {
+	events [eventsPerBuf]Event
+	num    int
+}
+
+type eventGenerator struct {
+	t           *Tracer
+	baseCtxInfo ctxInfo
+	buf         eventBuf
+}
+
+func makeEventGenerator(ctx context.Context, t *Tracer) eventGenerator {
+	return eventGenerator{
+		t:           t,
+		baseCtxInfo: infoFromCtx(ctx),
+	}
+}
+
+func (g *eventGenerator) flush() {
+	if g.buf.num > 0 {
+		g.t.workerDataCh <- g.buf
+		g.buf.num = 0
+	}
+}
+
+func (g *eventGenerator) add(ctx context.Context, e Event) {
+	e.StartUnixNano = time.Now().UnixNano()
+	info := infoFromCtx(ctx)
+	info = mergeCtxInfo(g.baseCtxInfo, info)
+	e.Reason = info.reason
+	e.BlockType = info.blockType
+	e.LevelPlusOne = info.levelPlusOne
+	if g.buf.num == eventsPerBuf {
+		g.flush()
+	}
+	g.buf.events[g.buf.num] = e
+	g.buf.num++
+}
+
+type workerState struct {
+	curFile          vfs.File
+	curBW            *bufio.Writer
+	numEntriesInFile int
+}
+
+func (t *Tracer) workerLoop() {
+	defer t.workerWait.Done()
+	stopCh := t.workerStopCh
+	dataCh := t.workerDataCh
+	var state workerState
+	t.workerNewFile(&state)
+	for {
+		select {
+		case <-stopCh:
+			close(dataCh)
+			// Flush any remaining traces.
+			for data := range dataCh {
+				t.workerWriteTraces(&state, data)
+			}
+			t.workerCloseFile(&state)
+			return
+
+		case data := <-dataCh:
+			t.workerWriteTraces(&state, data)
+		}
+	}
+}
+
+func (t *Tracer) workerWriteTraces(state *workerState, data eventBuf) {
+	if state.numEntriesInFile >= targetEntriesPerFile {
+		t.workerCloseFile(state)
+		t.workerNewFile(state)
+	}
+	state.numEntriesInFile += data.num
+	p := unsafe.Pointer(&data.events[0])
+	b := unsafe.Slice((*byte)(p), eventSize*data.num)
+	if _, err := state.curBW.Write(b); err != nil {
+		panic(err)
+	}
+}
+
+func (t *Tracer) workerNewFile(state *workerState) {
+	filename := fmt.Sprintf("IOTRACES-%s", time.Now().UTC().Format(time.RFC3339Nano))
+
+	file, err := t.fs.Create(t.fs.PathJoin(t.fsDir, filename))
+	if err != nil {
+		panic(err)
+	}
+	file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{
+		BytesPerSync: bytesPerFileSync,
+	})
+	state.curFile = file
+	state.curBW = bufio.NewWriter(file)
+	state.numEntriesInFile = 0
+}
+
+func (t *Tracer) workerCloseFile(state *workerState) {
+	if state.curFile != nil {
+		if err := state.curBW.Flush(); err != nil {
+			panic(err)
+		}
+		if err := state.curFile.Sync(); err != nil {
+			panic(err)
+		}
+		if err := state.curFile.Close(); err != nil {
+			panic(err)
+		}
+		state.curFile = nil
+		state.curBW = nil
+	}
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go

@@ -0,0 +1,522 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"io"
+	"os"
+	"sort"
+	"sync"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/errors/oserror"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// provider is the implementation of objstorage.Provider.
+type provider struct {
+	st Settings
+
+	fsDir vfs.File
+
+	tracer *objiotracing.Tracer
+
+	remote remoteSubsystem
+
+	mu struct {
+		sync.RWMutex
+
+		remote struct {
+			// catalogBatch accumulates remote object creations and deletions until
+			// Sync is called.
+			catalogBatch remoteobjcat.Batch
+
+			storageObjects map[remote.Locator]remote.Storage
+		}
+
+		// localObjectsChanged is set if non-remote objects were created or deleted
+		// but Sync was not yet called.
+		localObjectsChanged bool
+
+		// knownObjects maintains information about objects that are known to the provider.
+		// It is initialized with the list of files in the manifest when we open a DB.
+		knownObjects map[base.DiskFileNum]objstorage.ObjectMetadata
+
+		// protectedObjects are objects that cannot be unreferenced because they
+		// have outstanding SharedObjectBackingHandles. The value is a count of outstanding handles
+		protectedObjects map[base.DiskFileNum]int
+	}
+}
+
+var _ objstorage.Provider = (*provider)(nil)
+
+// Settings that must be specified when creating the provider.
+type Settings struct {
+	Logger base.Logger
+
+	// Local filesystem configuration.
+	FS        vfs.FS
+	FSDirName string
+
+	// FSDirInitialListing is a listing of FSDirName at the time of calling Open.
+	//
+	// This is an optional optimization to avoid double listing on Open when the
+	// higher layer already has a listing. When nil, we obtain the listing on
+	// Open.
+	FSDirInitialListing []string
+
+	// Cleaner cleans obsolete files from the local filesystem.
+	//
+	// The default cleaner uses the DeleteCleaner.
+	FSCleaner base.Cleaner
+
+	// NoSyncOnClose decides whether the implementation will enforce a
+	// close-time synchronization (e.g., fdatasync() or sync_file_range())
+	// on files it writes to. Setting this to true removes the guarantee for a
+	// sync on close. Some implementations can still issue a non-blocking sync.
+	NoSyncOnClose bool
+
+	// BytesPerSync enables periodic syncing of files in order to smooth out
+	// writes to disk. This option does not provide any persistence guarantee, but
+	// is used to avoid latency spikes if the OS automatically decides to write
+	// out a large chunk of dirty filesystem buffers.
+	BytesPerSync int
+
+	// Fields here are set only if the provider is to support remote objects
+	// (experimental).
+	Remote struct {
+		StorageFactory remote.StorageFactory
+
+		// If CreateOnShared is non-zero, sstables are created on remote storage using
+		// the CreateOnSharedLocator (when the PreferSharedStorage create option is
+		// true).
+		CreateOnShared        remote.CreateOnSharedStrategy
+		CreateOnSharedLocator remote.Locator
+
+		// CacheSizeBytes is the size of the on-disk block cache for objects
+		// on remote storage. If it is 0, no cache is used.
+		CacheSizeBytes int64
+
+		// CacheBlockSize is the block size of the cache; if 0, the default of 32KB is used.
+		CacheBlockSize int
+
+		// ShardingBlockSize is the size of a shard block. The cache is split into contiguous
+		// ShardingBlockSize units. The units are distributed across multiple independent shards
+		// of the cache, via a hash(offset) modulo num shards operation. The cache replacement
+		// policies operate at the level of shard, not whole cache. This is done to reduce lock
+		// contention.
+		//
+		// If ShardingBlockSize is 0, the default of 1 MB is used.
+		ShardingBlockSize int64
+
+		// The number of independent shards the cache leverages. Each shard is the same size,
+		// and a hash of filenum & offset map a read to a certain shard. If set to 0,
+		// 2*runtime.GOMAXPROCS is used as the shard count.
+		CacheShardCount int
+
+		// TODO(radu): allow the cache to live on another FS/location (e.g. to use
+		// instance-local SSD).
+	}
+}
+
+// DefaultSettings initializes default settings (with no remote storage),
+// suitable for tests and tools.
+func DefaultSettings(fs vfs.FS, dirName string) Settings {
+	return Settings{
+		Logger:        base.DefaultLogger,
+		FS:            fs,
+		FSDirName:     dirName,
+		FSCleaner:     base.DeleteCleaner{},
+		NoSyncOnClose: false,
+		BytesPerSync:  512 * 1024, // 512KB
+	}
+}
+
+// Open creates the provider.
+func Open(settings Settings) (objstorage.Provider, error) {
+	// Note: we can't just `return open(settings)` because in an error case we
+	// would return (*provider)(nil) which is not objstorage.Provider(nil).
+	p, err := open(settings)
+	if err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+func open(settings Settings) (p *provider, _ error) {
+	fsDir, err := settings.FS.OpenDir(settings.FSDirName)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if p == nil {
+			fsDir.Close()
+		}
+	}()
+
+	p = &provider{
+		st:    settings,
+		fsDir: fsDir,
+	}
+	p.mu.knownObjects = make(map[base.DiskFileNum]objstorage.ObjectMetadata)
+	p.mu.protectedObjects = make(map[base.DiskFileNum]int)
+
+	if objiotracing.Enabled {
+		p.tracer = objiotracing.Open(settings.FS, settings.FSDirName)
+	}
+
+	// Add local FS objects.
+	if err := p.vfsInit(); err != nil {
+		return nil, err
+	}
+
+	// Initialize remote subsystem (if configured) and add remote objects.
+	if err := p.remoteInit(); err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}
+
+// Close is part of the objstorage.Provider interface.
+func (p *provider) Close() error {
+	err := p.sharedClose()
+	if p.fsDir != nil {
+		err = firstError(err, p.fsDir.Close())
+		p.fsDir = nil
+	}
+	if objiotracing.Enabled {
+		if p.tracer != nil {
+			p.tracer.Close()
+			p.tracer = nil
+		}
+	}
+	return err
+}
+
+// OpenForReading opens an existing object.
+func (p *provider) OpenForReading(
+	ctx context.Context,
+	fileType base.FileType,
+	fileNum base.DiskFileNum,
+	opts objstorage.OpenOptions,
+) (objstorage.Readable, error) {
+	meta, err := p.Lookup(fileType, fileNum)
+	if err != nil {
+		if opts.MustExist {
+			p.st.Logger.Fatalf("%v", err)
+		}
+		return nil, err
+	}
+
+	var r objstorage.Readable
+	if !meta.IsRemote() {
+		r, err = p.vfsOpenForReading(ctx, fileType, fileNum, opts)
+	} else {
+		r, err = p.remoteOpenForReading(ctx, meta, opts)
+		if err != nil && p.isNotExistError(meta, err) {
+			// Wrap the error so that IsNotExistError functions properly.
+			err = errors.Mark(err, os.ErrNotExist)
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+	if objiotracing.Enabled {
+		r = p.tracer.WrapReadable(ctx, r, fileNum)
+	}
+	return r, nil
+}
+
+// Create creates a new object and opens it for writing.
+//
+// The object is not guaranteed to be durable (accessible in case of crashes)
+// until Sync is called.
+func (p *provider) Create(
+	ctx context.Context,
+	fileType base.FileType,
+	fileNum base.DiskFileNum,
+	opts objstorage.CreateOptions,
+) (w objstorage.Writable, meta objstorage.ObjectMetadata, err error) {
+	if opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone {
+		w, meta, err = p.sharedCreate(ctx, fileType, fileNum, p.st.Remote.CreateOnSharedLocator, opts)
+	} else {
+		w, meta, err = p.vfsCreate(ctx, fileType, fileNum)
+	}
+	if err != nil {
+		err = errors.Wrapf(err, "creating object %s", errors.Safe(fileNum))
+		return nil, objstorage.ObjectMetadata{}, err
+	}
+	p.addMetadata(meta)
+	if objiotracing.Enabled {
+		w = p.tracer.WrapWritable(ctx, w, fileNum)
+	}
+	return w, meta, nil
+}
+
+// Remove removes an object.
+//
+// Note that if the object is remote, the object is only (conceptually) removed
+// from this provider. If other providers have references on the remote object,
+// it will not be removed.
+//
+// The object is not guaranteed to be durably removed until Sync is called.
+func (p *provider) Remove(fileType base.FileType, fileNum base.DiskFileNum) error {
+	meta, err := p.Lookup(fileType, fileNum)
+	if err != nil {
+		return err
+	}
+
+	if !meta.IsRemote() {
+		err = p.vfsRemove(fileType, fileNum)
+	} else {
+		// TODO(radu): implement remote object removal (i.e. deref).
+		err = p.sharedUnref(meta)
+		if err != nil && p.isNotExistError(meta, err) {
+			// Wrap the error so that IsNotExistError functions properly.
+			err = errors.Mark(err, os.ErrNotExist)
+		}
+	}
+	if err != nil && !p.IsNotExistError(err) {
+		// We want to be able to retry a Remove, so we keep the object in our list.
+		// TODO(radu): we should mark the object as "zombie" and not allow any other
+		// operations.
+		return errors.Wrapf(err, "removing object %s", errors.Safe(fileNum))
+	}
+
+	p.removeMetadata(fileNum)
+	return err
+}
+
+func (p *provider) isNotExistError(meta objstorage.ObjectMetadata, err error) bool {
+	if meta.Remote.Storage != nil {
+		return meta.Remote.Storage.IsNotExistError(err)
+	}
+	return oserror.IsNotExist(err)
+}
+
+// IsNotExistError is part of the objstorage.Provider interface.
+func (p *provider) IsNotExistError(err error) bool {
+	// We use errors.Mark(err, os.ErrNotExist) for not-exist errors coming from
+	// remote.Storage.
+	return oserror.IsNotExist(err)
+}
+
+// Sync flushes the metadata from creation or removal of objects since the last Sync.
+func (p *provider) Sync() error {
+	if err := p.vfsSync(); err != nil {
+		return err
+	}
+	if err := p.sharedSync(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// LinkOrCopyFromLocal creates a new object that is either a copy of a given
+// local file or a hard link (if the new object is created on the same FS, and
+// if the FS supports it).
+//
+// The object is not guaranteed to be durable (accessible in case of crashes)
+// until Sync is called.
+func (p *provider) LinkOrCopyFromLocal(
+	ctx context.Context,
+	srcFS vfs.FS,
+	srcFilePath string,
+	dstFileType base.FileType,
+	dstFileNum base.DiskFileNum,
+	opts objstorage.CreateOptions,
+) (objstorage.ObjectMetadata, error) {
+	shared := opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone
+	if !shared && srcFS == p.st.FS {
+		// Wrap the normal filesystem with one which wraps newly created files with
+		// vfs.NewSyncingFile.
+		fs := vfs.NewSyncingFS(p.st.FS, vfs.SyncingFileOptions{
+			NoSyncOnClose: p.st.NoSyncOnClose,
+			BytesPerSync:  p.st.BytesPerSync,
+		})
+		dstPath := p.vfsPath(dstFileType, dstFileNum)
+		if err := vfs.LinkOrCopy(fs, srcFilePath, dstPath); err != nil {
+			return objstorage.ObjectMetadata{}, err
+		}
+
+		meta := objstorage.ObjectMetadata{
+			DiskFileNum: dstFileNum,
+			FileType:    dstFileType,
+		}
+		p.addMetadata(meta)
+		return meta, nil
+	}
+	// Create the object and copy the data.
+	w, meta, err := p.Create(ctx, dstFileType, dstFileNum, opts)
+	if err != nil {
+		return objstorage.ObjectMetadata{}, err
+	}
+	f, err := srcFS.Open(srcFilePath, vfs.SequentialReadsOption)
+	if err != nil {
+		return objstorage.ObjectMetadata{}, err
+	}
+	defer f.Close()
+	buf := make([]byte, 64*1024)
+	for {
+		n, readErr := f.Read(buf)
+		if readErr != nil && readErr != io.EOF {
+			w.Abort()
+			return objstorage.ObjectMetadata{}, readErr
+		}
+
+		if n > 0 {
+			if err := w.Write(buf[:n]); err != nil {
+				w.Abort()
+				return objstorage.ObjectMetadata{}, err
+			}
+		}
+
+		if readErr == io.EOF {
+			break
+		}
+	}
+	if err := w.Finish(); err != nil {
+		return objstorage.ObjectMetadata{}, err
+	}
+	return meta, nil
+}
+
+// Lookup is part of the objstorage.Provider interface.
+func (p *provider) Lookup(
+	fileType base.FileType, fileNum base.DiskFileNum,
+) (objstorage.ObjectMetadata, error) {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	meta, ok := p.mu.knownObjects[fileNum]
+	if !ok {
+		return objstorage.ObjectMetadata{}, errors.Wrapf(
+			os.ErrNotExist,
+			"file %s (type %d) unknown to the objstorage provider",
+			errors.Safe(fileNum), errors.Safe(fileType),
+		)
+	}
+	if meta.FileType != fileType {
+		return objstorage.ObjectMetadata{}, errors.AssertionFailedf(
+			"file %s type mismatch (known type %d, expected type %d)",
+			errors.Safe(fileNum), errors.Safe(meta.FileType), errors.Safe(fileType),
+		)
+	}
+	return meta, nil
+}
+
+// Path is part of the objstorage.Provider interface.
+func (p *provider) Path(meta objstorage.ObjectMetadata) string {
+	if !meta.IsRemote() {
+		return p.vfsPath(meta.FileType, meta.DiskFileNum)
+	}
+	return p.remotePath(meta)
+}
+
+// Size returns the size of the object.
+func (p *provider) Size(meta objstorage.ObjectMetadata) (int64, error) {
+	if !meta.IsRemote() {
+		return p.vfsSize(meta.FileType, meta.DiskFileNum)
+	}
+	return p.remoteSize(meta)
+}
+
+// List is part of the objstorage.Provider interface.
+func (p *provider) List() []objstorage.ObjectMetadata {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	res := make([]objstorage.ObjectMetadata, 0, len(p.mu.knownObjects))
+	for _, meta := range p.mu.knownObjects {
+		res = append(res, meta)
+	}
+	sort.Slice(res, func(i, j int) bool {
+		return res[i].DiskFileNum.FileNum() < res[j].DiskFileNum.FileNum()
+	})
+	return res
+}
+
+// Metrics is part of the objstorage.Provider interface.
+func (p *provider) Metrics() sharedcache.Metrics {
+	if p.remote.cache != nil {
+		return p.remote.cache.Metrics()
+	}
+	return sharedcache.Metrics{}
+}
+
+func (p *provider) addMetadata(meta objstorage.ObjectMetadata) {
+	if invariants.Enabled {
+		meta.AssertValid()
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.mu.knownObjects[meta.DiskFileNum] = meta
+	if meta.IsRemote() {
+		p.mu.remote.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
+			FileNum:        meta.DiskFileNum,
+			FileType:       meta.FileType,
+			CreatorID:      meta.Remote.CreatorID,
+			CreatorFileNum: meta.Remote.CreatorFileNum,
+			Locator:        meta.Remote.Locator,
+			CleanupMethod:  meta.Remote.CleanupMethod,
+		})
+	} else {
+		p.mu.localObjectsChanged = true
+	}
+}
+
+func (p *provider) removeMetadata(fileNum base.DiskFileNum) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	meta, ok := p.mu.knownObjects[fileNum]
+	if !ok {
+		return
+	}
+	delete(p.mu.knownObjects, fileNum)
+	if meta.IsRemote() {
+		p.mu.remote.catalogBatch.DeleteObject(fileNum)
+	} else {
+		p.mu.localObjectsChanged = true
+	}
+}
+
+// protectObject prevents the unreferencing of a remote object until
+// unprotectObject is called.
+func (p *provider) protectObject(fileNum base.DiskFileNum) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.mu.protectedObjects[fileNum] = p.mu.protectedObjects[fileNum] + 1
+}
+
+func (p *provider) unprotectObject(fileNum base.DiskFileNum) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	v := p.mu.protectedObjects[fileNum]
+	if invariants.Enabled && v == 0 {
+		panic("invalid protection count")
+	}
+	if v > 1 {
+		p.mu.protectedObjects[fileNum] = v - 1
+	} else {
+		delete(p.mu.protectedObjects, fileNum)
+		// TODO(radu): check if the object is still in knownObject; if not, unref it
+		// now.
+	}
+}
+
+func (p *provider) isProtected(fileNum base.DiskFileNum) bool {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.mu.protectedObjects[fileNum] > 0
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go

@@ -2,7 +2,7 @@
 // of this source code is governed by a BSD-style license that can be found in
 // the LICENSE file.
 
-package objstorage
+package objstorageprovider
 
 const (
 	// Constants for dynamic readahead of data blocks. Note that the size values
@@ -11,15 +11,15 @@ const (
 	minFileReadsForReadahead = 2
 	// TODO(bilal): Have the initial size value be a factor of the block size,
 	// as opposed to a hardcoded value.
-	initialReadaheadSize = 64 << 10  /* 64KB */
+	initialReadaheadSize = 64 << 10 /* 64KB */
-	maxReadaheadSize     = 256 << 10 /* 256KB */
 )
 
 // readaheadState contains state variables related to readahead. Updated on
 // file reads.
 type readaheadState struct {
 	// Number of sequential reads.
-	numReads int64
+	numReads         int64
+	maxReadaheadSize int64
 	// Size issued to the next call to Prefetch. Starts at or above
 	// initialReadaheadSize and grows exponentially until maxReadaheadSize.
 	size int64
@@ -32,22 +32,23 @@ type readaheadState struct {
 	limit int64
 }
 
-func makeReadaheadState() readaheadState {
+func makeReadaheadState(maxReadaheadSize int64) readaheadState {
 	return readaheadState{
-		size: initialReadaheadSize,
+		size:             initialReadaheadSize,
+		maxReadaheadSize: maxReadaheadSize,
 	}
 }
 
 func (rs *readaheadState) recordCacheHit(offset, blockLength int64) {
 	currentReadEnd := offset + blockLength
 	if rs.numReads >= minFileReadsForReadahead {
-		if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
+		if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
 			// This is a read that would have resulted in a readahead, had it
 			// not been a cache hit.
 			rs.limit = currentReadEnd
 			return
 		}
-		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize {
+		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+rs.maxReadaheadSize {
 			// We read too far away from rs.limit to benefit from readahead in
 			// any scenario. Reset all variables.
 			rs.numReads = 1
@@ -61,7 +62,7 @@ func (rs *readaheadState) recordCacheHit(offset, blockLength int64) {
 		// readahead.
 		return
 	}
-	if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
+	if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
 		// Blocks are being read sequentially and would benefit from readahead
 		// down the line.
 		rs.numReads++
@@ -89,28 +90,28 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 		// as well as the interval where a read would benefit from read ahead:
 		// [rs.limit, rs.limit + rs.size]
 		// We increase the latter interval to
-		// [rs.limit, rs.limit + maxReadaheadSize] to account for cases where
+		// [rs.limit, rs.limit + rs.maxReadaheadSize] to account for cases where
 		// readahead may not be beneficial with a small readahead size, but over
 		// time the readahead size would increase exponentially to make it
 		// beneficial.
-		if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
+		if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
 			// We are doing a read in the interval ahead of
 			// the last readahead range. In the diagrams below, ++++ is the last
 			// readahead range, ==== is the range represented by
-			// [rs.limit, rs.limit + maxReadaheadSize], and ---- is the range
+			// [rs.limit, rs.limit + rs.maxReadaheadSize], and ---- is the range
 			// being read.
 			//
-			//               rs.limit           rs.limit + maxReadaheadSize
+			//               rs.limit           rs.limit + rs.maxReadaheadSize
 			//         ++++++++++|===========================|
 			//
 			//              |-------------|
 			//            offset       currentReadEnd
 			//
 			// This case is also possible, as are all cases with an overlap
-			// between [rs.limit, rs.limit + maxReadaheadSize] and [offset,
+			// between [rs.limit, rs.limit + rs.maxReadaheadSize] and [offset,
 			// currentReadEnd]:
 			//
-			//               rs.limit           rs.limit + maxReadaheadSize
+			//               rs.limit           rs.limit + rs.maxReadaheadSize
 			//         ++++++++++|===========================|
 			//
 			//                                            |-------------|
@@ -122,19 +123,19 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 			rs.prevSize = rs.size
 			// Increase rs.size for the next read.
 			rs.size *= 2
-			if rs.size > maxReadaheadSize {
+			if rs.size > rs.maxReadaheadSize {
-				rs.size = maxReadaheadSize
+				rs.size = rs.maxReadaheadSize
 			}
 			return rs.prevSize
 		}
-		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize {
+		if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+rs.maxReadaheadSize {
 			// The above conditional has rs.limit > rs.prevSize to confirm that
 			// rs.limit - rs.prevSize would not underflow.
 			// We read too far away from rs.limit to benefit from readahead in
 			// any scenario. Reset all variables.
 			// The case where we read too far ahead:
 			//
-			// (rs.limit - rs.prevSize)    (rs.limit)   (rs.limit + maxReadaheadSize)
+			// (rs.limit - rs.prevSize)    (rs.limit)   (rs.limit + rs.maxReadaheadSize)
 			//                    |+++++++++++++|=============|
 			//
 			//                                                  |-------------|
@@ -142,7 +143,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 			//
 			// Or too far behind:
 			//
-			// (rs.limit - rs.prevSize)    (rs.limit)   (rs.limit + maxReadaheadSize)
+			// (rs.limit - rs.prevSize)    (rs.limit)   (rs.limit + rs.maxReadaheadSize)
 			//                    |+++++++++++++|=============|
 			//
 			//    |-------------|
@@ -160,7 +161,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 		// readahead, but there's no reason to issue a readahead call at the
 		// moment.
 		//
-		// (rs.limit - rs.prevSize)            (rs.limit + maxReadaheadSize)
+		// (rs.limit - rs.prevSize)            (rs.limit + rs.maxReadaheadSize)
 		//                    |+++++++++++++|===============|
 		//                             (rs.limit)
 		//
@@ -170,11 +171,11 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 		rs.numReads++
 		return 0
 	}
-	if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize {
+	if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize {
 		// Blocks are being read sequentially and would benefit from readahead
 		// down the line.
 		//
-		//                       (rs.limit)   (rs.limit + maxReadaheadSize)
+		//                       (rs.limit)   (rs.limit + rs.maxReadaheadSize)
 		//                         |=============|
 		//
 		//                    |-------|
@@ -186,7 +187,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
 	// We read too far ahead of the last read, or before it. This indicates
 	// a random read, where readahead is not desirable. Reset all variables.
 	//
-	// (rs.limit - maxReadaheadSize)  (rs.limit)   (rs.limit + maxReadaheadSize)
+	// (rs.limit - rs.maxReadaheadSize)  (rs.limit)   (rs.limit + rs.maxReadaheadSize)
 	//                     |+++++++++++++|=============|
 	//
 	//                                                    |-------|

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote.go

@@ -0,0 +1,376 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"sync"
+	"sync/atomic"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+)
+
+// remoteSubsystem contains the provider fields related to remote storage.
+// All fields remain unset if remote storage is not configured.
+type remoteSubsystem struct {
+	catalog *remoteobjcat.Catalog
+	// catalogSyncMutex is used to correctly serialize two sharedSync operations.
+	// It must be acquired before the provider mutex.
+	catalogSyncMutex sync.Mutex
+
+	cache *sharedcache.Cache
+
+	// shared contains the fields relevant to shared objects, i.e. objects that
+	// are created by Pebble and potentially shared between Pebble instances.
+	shared struct {
+		// initialized guards access to the creatorID field.
+		initialized atomic.Bool
+		creatorID   objstorage.CreatorID
+		initOnce    sync.Once
+
+		// checkRefsOnOpen controls whether we check the ref marker file when opening
+		// an object. Normally this is true when invariants are enabled (but the provider
+		// test tweaks this field).
+		checkRefsOnOpen bool
+	}
+}
+
+// remoteInit initializes the remote object subsystem (if configured) and finds
+// any remote objects.
+func (p *provider) remoteInit() error {
+	if p.st.Remote.StorageFactory == nil {
+		return nil
+	}
+	catalog, contents, err := remoteobjcat.Open(p.st.FS, p.st.FSDirName)
+	if err != nil {
+		return errors.Wrapf(err, "pebble: could not open remote object catalog")
+	}
+	p.remote.catalog = catalog
+	p.remote.shared.checkRefsOnOpen = invariants.Enabled
+
+	// The creator ID may or may not be initialized yet.
+	if contents.CreatorID.IsSet() {
+		p.remote.initShared(contents.CreatorID)
+		p.st.Logger.Infof("remote storage configured; creatorID = %s", contents.CreatorID)
+	} else {
+		p.st.Logger.Infof("remote storage configured; no creatorID yet")
+	}
+
+	if p.st.Remote.CacheSizeBytes > 0 {
+		const defaultBlockSize = 32 * 1024
+		blockSize := p.st.Remote.CacheBlockSize
+		if blockSize == 0 {
+			blockSize = defaultBlockSize
+		}
+
+		const defaultShardingBlockSize = 1024 * 1024
+		shardingBlockSize := p.st.Remote.ShardingBlockSize
+		if shardingBlockSize == 0 {
+			shardingBlockSize = defaultShardingBlockSize
+		}
+
+		numShards := p.st.Remote.CacheShardCount
+		if numShards == 0 {
+			numShards = 2 * runtime.GOMAXPROCS(0)
+		}
+
+		p.remote.cache, err = sharedcache.Open(
+			p.st.FS, p.st.Logger, p.st.FSDirName, blockSize, shardingBlockSize, p.st.Remote.CacheSizeBytes, numShards)
+		if err != nil {
+			return errors.Wrapf(err, "pebble: could not open remote object cache")
+		}
+	}
+
+	for _, meta := range contents.Objects {
+		o := objstorage.ObjectMetadata{
+			DiskFileNum: meta.FileNum,
+			FileType:    meta.FileType,
+		}
+		o.Remote.CreatorID = meta.CreatorID
+		o.Remote.CreatorFileNum = meta.CreatorFileNum
+		o.Remote.CleanupMethod = meta.CleanupMethod
+		o.Remote.Locator = meta.Locator
+		o.Remote.CustomObjectName = meta.CustomObjectName
+		o.Remote.Storage, err = p.ensureStorageLocked(o.Remote.Locator)
+		if err != nil {
+			return errors.Wrapf(err, "creating remote.Storage object for locator '%s'", o.Remote.Locator)
+		}
+		if invariants.Enabled {
+			o.AssertValid()
+		}
+		p.mu.knownObjects[o.DiskFileNum] = o
+	}
+	return nil
+}
+
+// initShared initializes the creator ID, allowing use of shared objects.
+func (ss *remoteSubsystem) initShared(creatorID objstorage.CreatorID) {
+	ss.shared.initOnce.Do(func() {
+		ss.shared.creatorID = creatorID
+		ss.shared.initialized.Store(true)
+	})
+}
+
+func (p *provider) sharedClose() error {
+	if p.st.Remote.StorageFactory == nil {
+		return nil
+	}
+	var err error
+	if p.remote.cache != nil {
+		err = p.remote.cache.Close()
+		p.remote.cache = nil
+	}
+	if p.remote.catalog != nil {
+		err = firstError(err, p.remote.catalog.Close())
+		p.remote.catalog = nil
+	}
+	return err
+}
+
+// SetCreatorID is part of the objstorage.Provider interface.
+func (p *provider) SetCreatorID(creatorID objstorage.CreatorID) error {
+	if p.st.Remote.StorageFactory == nil {
+		return errors.AssertionFailedf("attempt to set CreatorID but remote storage not enabled")
+	}
+	// Note: this call is a cheap no-op if the creator ID was already set. This
+	// call also checks if we are trying to change the ID.
+	if err := p.remote.catalog.SetCreatorID(creatorID); err != nil {
+		return err
+	}
+	if !p.remote.shared.initialized.Load() {
+		p.st.Logger.Infof("remote storage creatorID set to %s", creatorID)
+		p.remote.initShared(creatorID)
+	}
+	return nil
+}
+
+// IsSharedForeign is part of the objstorage.Provider interface.
+func (p *provider) IsSharedForeign(meta objstorage.ObjectMetadata) bool {
+	if !p.remote.shared.initialized.Load() {
+		return false
+	}
+	return meta.IsShared() && (meta.Remote.CreatorID != p.remote.shared.creatorID)
+}
+
+func (p *provider) remoteCheckInitialized() error {
+	if p.st.Remote.StorageFactory == nil {
+		return errors.Errorf("remote object support not configured")
+	}
+	return nil
+}
+
+func (p *provider) sharedCheckInitialized() error {
+	if err := p.remoteCheckInitialized(); err != nil {
+		return err
+	}
+	if !p.remote.shared.initialized.Load() {
+		return errors.Errorf("remote object support not available: remote creator ID not yet set")
+	}
+	return nil
+}
+
+func (p *provider) sharedSync() error {
+	// Serialize parallel sync operations. Note that ApplyBatch is already
+	// serialized internally, but we want to make sure they get called with
+	// batches in the right order.
+	p.remote.catalogSyncMutex.Lock()
+	defer p.remote.catalogSyncMutex.Unlock()
+
+	batch := func() remoteobjcat.Batch {
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		res := p.mu.remote.catalogBatch.Copy()
+		p.mu.remote.catalogBatch.Reset()
+		return res
+	}()
+
+	if batch.IsEmpty() {
+		return nil
+	}
+
+	if err := p.remote.catalog.ApplyBatch(batch); err != nil {
+		// Put back the batch (for the next Sync), appending any operations that
+		// happened in the meantime.
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		batch.Append(p.mu.remote.catalogBatch)
+		p.mu.remote.catalogBatch = batch
+		return err
+	}
+
+	return nil
+}
+
+func (p *provider) remotePath(meta objstorage.ObjectMetadata) string {
+	if meta.Remote.Locator != "" {
+		return fmt.Sprintf("remote-%s://%s", meta.Remote.Locator, remoteObjectName(meta))
+	}
+	return "remote://" + remoteObjectName(meta)
+}
+
+// sharedCreateRef creates a reference marker object.
+func (p *provider) sharedCreateRef(meta objstorage.ObjectMetadata) error {
+	if err := p.sharedCheckInitialized(); err != nil {
+		return err
+	}
+	if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
+		return nil
+	}
+	refName := p.sharedObjectRefName(meta)
+	writer, err := meta.Remote.Storage.CreateObject(refName)
+	if err == nil {
+		// The object is empty, just close the writer.
+		err = writer.Close()
+	}
+	if err != nil {
+		return errors.Wrapf(err, "creating marker object %q", refName)
+	}
+	return nil
+}
+
+func (p *provider) sharedCreate(
+	_ context.Context,
+	fileType base.FileType,
+	fileNum base.DiskFileNum,
+	locator remote.Locator,
+	opts objstorage.CreateOptions,
+) (objstorage.Writable, objstorage.ObjectMetadata, error) {
+	if err := p.sharedCheckInitialized(); err != nil {
+		return nil, objstorage.ObjectMetadata{}, err
+	}
+	storage, err := p.ensureStorage(locator)
+	if err != nil {
+		return nil, objstorage.ObjectMetadata{}, err
+	}
+	meta := objstorage.ObjectMetadata{
+		DiskFileNum: fileNum,
+		FileType:    fileType,
+	}
+	meta.Remote.CreatorID = p.remote.shared.creatorID
+	meta.Remote.CreatorFileNum = fileNum
+	meta.Remote.CleanupMethod = opts.SharedCleanupMethod
+	meta.Remote.Locator = locator
+	meta.Remote.Storage = storage
+
+	objName := remoteObjectName(meta)
+	writer, err := storage.CreateObject(objName)
+	if err != nil {
+		return nil, objstorage.ObjectMetadata{}, errors.Wrapf(err, "creating object %q", objName)
+	}
+	return &sharedWritable{
+		p:             p,
+		meta:          meta,
+		storageWriter: writer,
+	}, meta, nil
+}
+
+func (p *provider) remoteOpenForReading(
+	ctx context.Context, meta objstorage.ObjectMetadata, opts objstorage.OpenOptions,
+) (objstorage.Readable, error) {
+	if err := p.remoteCheckInitialized(); err != nil {
+		return nil, err
+	}
+	// Verify we have a reference on this object; for performance reasons, we only
+	// do this in testing scenarios.
+	if p.remote.shared.checkRefsOnOpen && meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
+		if err := p.sharedCheckInitialized(); err != nil {
+			return nil, err
+		}
+		refName := p.sharedObjectRefName(meta)
+		if _, err := meta.Remote.Storage.Size(refName); err != nil {
+			if meta.Remote.Storage.IsNotExistError(err) {
+				if opts.MustExist {
+					p.st.Logger.Fatalf("marker object %q does not exist", refName)
+					// TODO(radu): maybe list references for the object.
+				}
+				return nil, errors.Errorf("marker object %q does not exist", refName)
+			}
+			return nil, errors.Wrapf(err, "checking marker object %q", refName)
+		}
+	}
+	objName := remoteObjectName(meta)
+	reader, size, err := meta.Remote.Storage.ReadObject(ctx, objName)
+	if err != nil {
+		if opts.MustExist && meta.Remote.Storage.IsNotExistError(err) {
+			p.st.Logger.Fatalf("object %q does not exist", objName)
+			// TODO(radu): maybe list references for the object.
+		}
+		return nil, err
+	}
+	return p.newRemoteReadable(reader, size, meta.DiskFileNum), nil
+}
+
+func (p *provider) remoteSize(meta objstorage.ObjectMetadata) (int64, error) {
+	if err := p.remoteCheckInitialized(); err != nil {
+		return 0, err
+	}
+	objName := remoteObjectName(meta)
+	return meta.Remote.Storage.Size(objName)
+}
+
+// sharedUnref implements object "removal" with the remote backend. The ref
+// marker object is removed and the backing object is removed only if there are
+// no other ref markers.
+func (p *provider) sharedUnref(meta objstorage.ObjectMetadata) error {
+	if meta.Remote.CleanupMethod == objstorage.SharedNoCleanup {
+		// Never delete objects in this mode.
+		return nil
+	}
+	if p.isProtected(meta.DiskFileNum) {
+		// TODO(radu): we need a mechanism to unref the object when it becomes
+		// unprotected.
+		return nil
+	}
+
+	refName := p.sharedObjectRefName(meta)
+	// Tolerate a not-exists error.
+	if err := meta.Remote.Storage.Delete(refName); err != nil && !meta.Remote.Storage.IsNotExistError(err) {
+		return err
+	}
+	otherRefs, err := meta.Remote.Storage.List(sharedObjectRefPrefix(meta), "" /* delimiter */)
+	if err != nil {
+		return err
+	}
+	if len(otherRefs) == 0 {
+		objName := remoteObjectName(meta)
+		if err := meta.Remote.Storage.Delete(objName); err != nil && !meta.Remote.Storage.IsNotExistError(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+// ensureStorageLocked populates the remote.Storage object for the given
+// locator, if necessary. p.mu must be held.
+func (p *provider) ensureStorageLocked(locator remote.Locator) (remote.Storage, error) {
+	if p.mu.remote.storageObjects == nil {
+		p.mu.remote.storageObjects = make(map[remote.Locator]remote.Storage)
+	}
+	if res, ok := p.mu.remote.storageObjects[locator]; ok {
+		return res, nil
+	}
+	res, err := p.st.Remote.StorageFactory.CreateStorage(locator)
+	if err != nil {
+		return nil, err
+	}
+
+	p.mu.remote.storageObjects[locator] = res
+	return res, nil
+}
+
+// ensureStorage populates the remote.Storage object for the given locator, if necessary.
+func (p *provider) ensureStorage(locator remote.Locator) (remote.Storage, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.ensureStorageLocked(locator)
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_backing.go

@@ -0,0 +1,303 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+)
+
+const (
+	tagCreatorID      = 1
+	tagCreatorFileNum = 2
+	tagCleanupMethod  = 3
+	// tagRefCheckID encodes the information for a ref marker that needs to be
+	// checked when attaching this object to another provider. This is set to the
+	// creator ID and FileNum for the provider that encodes the backing, and
+	// allows the "target" provider to check that the "source" provider kept its
+	// reference on the object alive.
+	tagRefCheckID = 4
+	// tagLocator encodes the remote.Locator; if absent the locator is "". It is
+	// followed by the locator string length and the locator string.
+	tagLocator = 5
+	// tagLocator encodes a custom object name (if present). It is followed by the
+	// custom name string length and the string.
+	tagCustomObjectName = 6
+
+	// Any new tags that don't have the tagNotSafeToIgnoreMask bit set must be
+	// followed by the length of the data (so they can be skipped).
+
+	// Any new tags that have the tagNotSafeToIgnoreMask bit set cause errors if
+	// they are encountered by earlier code that doesn't know the tag.
+	tagNotSafeToIgnoreMask = 64
+)
+
+func (p *provider) encodeRemoteObjectBacking(
+	meta *objstorage.ObjectMetadata,
+) (objstorage.RemoteObjectBacking, error) {
+	if !meta.IsRemote() {
+		return nil, errors.AssertionFailedf("object %s not on remote storage", meta.DiskFileNum)
+	}
+
+	buf := make([]byte, 0, binary.MaxVarintLen64*4)
+	buf = binary.AppendUvarint(buf, tagCreatorID)
+	buf = binary.AppendUvarint(buf, uint64(meta.Remote.CreatorID))
+	// TODO(radu): encode file type as well?
+	buf = binary.AppendUvarint(buf, tagCreatorFileNum)
+	buf = binary.AppendUvarint(buf, uint64(meta.Remote.CreatorFileNum.FileNum()))
+	buf = binary.AppendUvarint(buf, tagCleanupMethod)
+	buf = binary.AppendUvarint(buf, uint64(meta.Remote.CleanupMethod))
+	if meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
+		buf = binary.AppendUvarint(buf, tagRefCheckID)
+		buf = binary.AppendUvarint(buf, uint64(p.remote.shared.creatorID))
+		buf = binary.AppendUvarint(buf, uint64(meta.DiskFileNum.FileNum()))
+	}
+	if meta.Remote.Locator != "" {
+		buf = binary.AppendUvarint(buf, tagLocator)
+		buf = encodeString(buf, string(meta.Remote.Locator))
+	}
+	if meta.Remote.CustomObjectName != "" {
+		buf = binary.AppendUvarint(buf, tagCustomObjectName)
+		buf = encodeString(buf, meta.Remote.CustomObjectName)
+	}
+	return buf, nil
+}
+
+type remoteObjectBackingHandle struct {
+	backing objstorage.RemoteObjectBacking
+	fileNum base.DiskFileNum
+	p       *provider
+}
+
+func (s *remoteObjectBackingHandle) Get() (objstorage.RemoteObjectBacking, error) {
+	if s.backing == nil {
+		return nil, errors.Errorf("RemoteObjectBackingHandle.Get() called after Close()")
+	}
+	return s.backing, nil
+}
+
+func (s *remoteObjectBackingHandle) Close() {
+	if s.backing != nil {
+		s.backing = nil
+		s.p.unprotectObject(s.fileNum)
+	}
+}
+
+var _ objstorage.RemoteObjectBackingHandle = (*remoteObjectBackingHandle)(nil)
+
+// RemoteObjectBacking is part of the objstorage.Provider interface.
+func (p *provider) RemoteObjectBacking(
+	meta *objstorage.ObjectMetadata,
+) (objstorage.RemoteObjectBackingHandle, error) {
+	backing, err := p.encodeRemoteObjectBacking(meta)
+	if err != nil {
+		return nil, err
+	}
+	p.protectObject(meta.DiskFileNum)
+	return &remoteObjectBackingHandle{
+		backing: backing,
+		fileNum: meta.DiskFileNum,
+		p:       p,
+	}, nil
+}
+
+// CreateExternalObjectBacking is part of the objstorage.Provider interface.
+func (p *provider) CreateExternalObjectBacking(
+	locator remote.Locator, objName string,
+) (objstorage.RemoteObjectBacking, error) {
+	var meta objstorage.ObjectMetadata
+	meta.Remote.Locator = locator
+	meta.Remote.CustomObjectName = objName
+	meta.Remote.CleanupMethod = objstorage.SharedNoCleanup
+	return p.encodeRemoteObjectBacking(&meta)
+}
+
+type decodedBacking struct {
+	meta objstorage.ObjectMetadata
+	// refToCheck is set only when meta.Remote.CleanupMethod is RefTracking
+	refToCheck struct {
+		creatorID objstorage.CreatorID
+		fileNum   base.DiskFileNum
+	}
+}
+
+// decodeRemoteObjectBacking decodes the remote object metadata.
+//
+// Note that the meta.Remote.Storage field is not set.
+func decodeRemoteObjectBacking(
+	fileType base.FileType, fileNum base.DiskFileNum, buf objstorage.RemoteObjectBacking,
+) (decodedBacking, error) {
+	var creatorID, creatorFileNum, cleanupMethod, refCheckCreatorID, refCheckFileNum uint64
+	var locator, customObjName string
+	br := bytes.NewReader(buf)
+	for {
+		tag, err := binary.ReadUvarint(br)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return decodedBacking{}, err
+		}
+		switch tag {
+		case tagCreatorID:
+			creatorID, err = binary.ReadUvarint(br)
+
+		case tagCreatorFileNum:
+			creatorFileNum, err = binary.ReadUvarint(br)
+
+		case tagCleanupMethod:
+			cleanupMethod, err = binary.ReadUvarint(br)
+
+		case tagRefCheckID:
+			refCheckCreatorID, err = binary.ReadUvarint(br)
+			if err == nil {
+				refCheckFileNum, err = binary.ReadUvarint(br)
+			}
+
+		case tagLocator:
+			locator, err = decodeString(br)
+
+		case tagCustomObjectName:
+			customObjName, err = decodeString(br)
+
+		default:
+			// Ignore unknown tags, unless they're not safe to ignore.
+			if tag&tagNotSafeToIgnoreMask != 0 {
+				return decodedBacking{}, errors.Newf("unknown tag %d", tag)
+			}
+			var dataLen uint64
+			dataLen, err = binary.ReadUvarint(br)
+			if err == nil {
+				_, err = br.Seek(int64(dataLen), io.SeekCurrent)
+			}
+		}
+		if err != nil {
+			return decodedBacking{}, err
+		}
+	}
+	if customObjName == "" {
+		if creatorID == 0 {
+			return decodedBacking{}, errors.Newf("remote object backing missing creator ID")
+		}
+		if creatorFileNum == 0 {
+			return decodedBacking{}, errors.Newf("remote object backing missing creator file num")
+		}
+	}
+	var res decodedBacking
+	res.meta.DiskFileNum = fileNum
+	res.meta.FileType = fileType
+	res.meta.Remote.CreatorID = objstorage.CreatorID(creatorID)
+	res.meta.Remote.CreatorFileNum = base.FileNum(creatorFileNum).DiskFileNum()
+	res.meta.Remote.CleanupMethod = objstorage.SharedCleanupMethod(cleanupMethod)
+	if res.meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
+		if refCheckCreatorID == 0 || refCheckFileNum == 0 {
+			return decodedBacking{}, errors.Newf("remote object backing missing ref to check")
+		}
+		res.refToCheck.creatorID = objstorage.CreatorID(refCheckCreatorID)
+		res.refToCheck.fileNum = base.FileNum(refCheckFileNum).DiskFileNum()
+	}
+	res.meta.Remote.Locator = remote.Locator(locator)
+	res.meta.Remote.CustomObjectName = customObjName
+	return res, nil
+}
+
+func encodeString(buf []byte, s string) []byte {
+	buf = binary.AppendUvarint(buf, uint64(len(s)))
+	buf = append(buf, []byte(s)...)
+	return buf
+}
+
+func decodeString(br io.ByteReader) (string, error) {
+	length, err := binary.ReadUvarint(br)
+	if err != nil || length == 0 {
+		return "", err
+	}
+	buf := make([]byte, length)
+	for i := range buf {
+		buf[i], err = br.ReadByte()
+		if err != nil {
+			return "", err
+		}
+	}
+	return string(buf), nil
+}
+
+// AttachRemoteObjects is part of the objstorage.Provider interface.
+func (p *provider) AttachRemoteObjects(
+	objs []objstorage.RemoteObjectToAttach,
+) ([]objstorage.ObjectMetadata, error) {
+	decoded := make([]decodedBacking, len(objs))
+	for i, o := range objs {
+		var err error
+		decoded[i], err = decodeRemoteObjectBacking(o.FileType, o.FileNum, o.Backing)
+		if err != nil {
+			return nil, err
+		}
+		decoded[i].meta.Remote.Storage, err = p.ensureStorage(decoded[i].meta.Remote.Locator)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Create the reference marker objects.
+	// TODO(radu): parallelize this.
+	for _, d := range decoded {
+		if d.meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
+			continue
+		}
+		if err := p.sharedCreateRef(d.meta); err != nil {
+			// TODO(radu): clean up references previously created in this loop.
+			return nil, err
+		}
+		// Check the "origin"'s reference.
+		refName := sharedObjectRefName(d.meta, d.refToCheck.creatorID, d.refToCheck.fileNum)
+		if _, err := d.meta.Remote.Storage.Size(refName); err != nil {
+			_ = p.sharedUnref(d.meta)
+			// TODO(radu): clean up references previously created in this loop.
+			if d.meta.Remote.Storage.IsNotExistError(err) {
+				return nil, errors.Errorf("origin marker object %q does not exist;"+
+					" object probably removed from the provider which created the backing", refName)
+			}
+			return nil, errors.Wrapf(err, "checking origin's marker object %s", refName)
+		}
+	}
+
+	func() {
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		for _, d := range decoded {
+			p.mu.remote.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
+				FileNum:        d.meta.DiskFileNum,
+				FileType:       d.meta.FileType,
+				CreatorID:      d.meta.Remote.CreatorID,
+				CreatorFileNum: d.meta.Remote.CreatorFileNum,
+				CleanupMethod:  d.meta.Remote.CleanupMethod,
+				Locator:        d.meta.Remote.Locator,
+			})
+		}
+	}()
+	if err := p.sharedSync(); err != nil {
+		return nil, err
+	}
+
+	metas := make([]objstorage.ObjectMetadata, len(decoded))
+	for i, d := range decoded {
+		metas[i] = d.meta
+	}
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	for _, meta := range metas {
+		p.mu.knownObjects[meta.DiskFileNum] = meta
+	}
+	return metas, nil
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_obj_name.go

@@ -0,0 +1,91 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"fmt"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+)
+
+// remoteObjectName returns the name of an object on remote storage.
+//
+// For sstables, the format is: <hash>-<creator-id>-<file-num>.sst
+// For example: 1a3f-2-000001.sst
+func remoteObjectName(meta objstorage.ObjectMetadata) string {
+	if meta.Remote.CustomObjectName != "" {
+		return meta.Remote.CustomObjectName
+	}
+	switch meta.FileType {
+	case base.FileTypeTable:
+		return fmt.Sprintf(
+			"%04x-%d-%06d.sst",
+			objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(),
+		)
+	}
+	panic("unknown FileType")
+}
+
+// sharedObjectRefName returns the name of the object's ref marker associated
+// with a given referencing provider. This name is the object's name concatenated with
+// ".ref.<ref-creator-id>.<local-file-num>".
+//
+// For example: 1a3f-2-000001.sst.ref.5.000008
+func sharedObjectRefName(
+	meta objstorage.ObjectMetadata, refCreatorID objstorage.CreatorID, refFileNum base.DiskFileNum,
+) string {
+	if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
+		panic("ref object used when ref tracking disabled")
+	}
+	if meta.Remote.CustomObjectName != "" {
+		return fmt.Sprintf(
+			"%s.ref.%d.%06d", meta.Remote.CustomObjectName, refCreatorID, refFileNum.FileNum(),
+		)
+	}
+	switch meta.FileType {
+	case base.FileTypeTable:
+		return fmt.Sprintf(
+			"%04x-%d-%06d.sst.ref.%d.%06d",
+			objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(), refCreatorID, refFileNum.FileNum(),
+		)
+	}
+	panic("unknown FileType")
+}
+
+func sharedObjectRefPrefix(meta objstorage.ObjectMetadata) string {
+	if meta.Remote.CustomObjectName != "" {
+		return meta.Remote.CustomObjectName + ".ref."
+	}
+	switch meta.FileType {
+	case base.FileTypeTable:
+		return fmt.Sprintf(
+			"%04x-%d-%06d.sst.ref.",
+			objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(),
+		)
+	}
+	panic("unknown FileType")
+}
+
+// sharedObjectRefName returns the name of the object's ref marker associated
+// with this provider. This name is the object's name concatenated with
+// ".ref.<creator-id>.<local-file-num>".
+//
+// For example: 1a3f-2-000001.sst.ref.5.000008
+func (p *provider) sharedObjectRefName(meta objstorage.ObjectMetadata) string {
+	if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
+		panic("ref object used when ref tracking disabled")
+	}
+	return sharedObjectRefName(meta, p.remote.shared.creatorID, meta.DiskFileNum)
+}
+
+// objHash returns a 16-bit hash value derived from the creator ID and creator
+// file num. We prepend this value to object names to ensure balanced
+// partitioning with AWS (and likely other blob storage providers).
+func objHash(meta objstorage.ObjectMetadata) uint16 {
+	const prime1 = 7459
+	const prime2 = 17539
+	return uint16(uint64(meta.Remote.CreatorID)*prime1 + uint64(meta.Remote.CreatorFileNum.FileNum())*prime2)
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_readable.go

@@ -0,0 +1,162 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"io"
+
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+)
+
+const remoteMaxReadaheadSize = 1024 * 1024 /* 1MB */
+
+// remoteReadable is a very simple implementation of Readable on top of the
+// ReadCloser returned by remote.Storage.CreateObject.
+type remoteReadable struct {
+	objReader remote.ObjectReader
+	size      int64
+	fileNum   base.DiskFileNum
+	provider  *provider
+}
+
+var _ objstorage.Readable = (*remoteReadable)(nil)
+
+func (p *provider) newRemoteReadable(
+	objReader remote.ObjectReader, size int64, fileNum base.DiskFileNum,
+) *remoteReadable {
+	return &remoteReadable{
+		objReader: objReader,
+		size:      size,
+		fileNum:   fileNum,
+		provider:  p,
+	}
+}
+
+// ReadAt is part of the objstorage.Readable interface.
+func (r *remoteReadable) ReadAt(ctx context.Context, p []byte, offset int64) error {
+	return r.readInternal(ctx, p, offset, false /* forCompaction */)
+}
+
+// readInternal performs a read for the object, using the cache when
+// appropriate.
+func (r *remoteReadable) readInternal(
+	ctx context.Context, p []byte, offset int64, forCompaction bool,
+) error {
+	if cache := r.provider.remote.cache; cache != nil {
+		flags := sharedcache.ReadFlags{
+			// Don't add data to the cache if this read is for a compaction.
+			ReadOnly: forCompaction,
+		}
+		return r.provider.remote.cache.ReadAt(ctx, r.fileNum, p, offset, r.objReader, r.size, flags)
+	}
+	return r.objReader.ReadAt(ctx, p, offset)
+}
+
+func (r *remoteReadable) Close() error {
+	defer func() { r.objReader = nil }()
+	return r.objReader.Close()
+}
+
+func (r *remoteReadable) Size() int64 {
+	return r.size
+}
+
+func (r *remoteReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
+	// TODO(radu): use a pool.
+	rh := &remoteReadHandle{readable: r}
+	rh.readahead.state = makeReadaheadState(remoteMaxReadaheadSize)
+	return rh
+}
+
+type remoteReadHandle struct {
+	readable  *remoteReadable
+	readahead struct {
+		state  readaheadState
+		data   []byte
+		offset int64
+	}
+	forCompaction bool
+}
+
+var _ objstorage.ReadHandle = (*remoteReadHandle)(nil)
+
+// ReadAt is part of the objstorage.ReadHandle interface.
+func (r *remoteReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) error {
+	readaheadSize := r.maybeReadahead(offset, len(p))
+
+	// Check if we already have the data from a previous read-ahead.
+	if rhSize := int64(len(r.readahead.data)); rhSize > 0 {
+		if r.readahead.offset <= offset && r.readahead.offset+rhSize > offset {
+			n := copy(p, r.readahead.data[offset-r.readahead.offset:])
+			if n == len(p) {
+				// All data was available.
+				return nil
+			}
+			// Use the data that we had and do a shorter read.
+			offset += int64(n)
+			p = p[n:]
+			readaheadSize -= n
+		}
+	}
+
+	if readaheadSize > len(p) {
+		// Don't try to read past EOF.
+		if offset+int64(readaheadSize) > r.readable.size {
+			readaheadSize = int(r.readable.size - offset)
+			if readaheadSize <= 0 {
+				// This shouldn't happen in practice (Pebble should never try to read
+				// past EOF).
+				return io.EOF
+			}
+		}
+		r.readahead.offset = offset
+		// TODO(radu): we need to somehow account for this memory.
+		if cap(r.readahead.data) >= readaheadSize {
+			r.readahead.data = r.readahead.data[:readaheadSize]
+		} else {
+			r.readahead.data = make([]byte, readaheadSize)
+		}
+
+		if err := r.readable.readInternal(ctx, r.readahead.data, offset, r.forCompaction); err != nil {
+			// Make sure we don't treat the data as valid next time.
+			r.readahead.data = r.readahead.data[:0]
+			return err
+		}
+		copy(p, r.readahead.data)
+		return nil
+	}
+
+	return r.readable.readInternal(ctx, p, offset, r.forCompaction)
+}
+
+func (r *remoteReadHandle) maybeReadahead(offset int64, len int) int {
+	if r.forCompaction {
+		return remoteMaxReadaheadSize
+	}
+	return int(r.readahead.state.maybeReadahead(offset, int64(len)))
+}
+
+// Close is part of the objstorage.ReadHandle interface.
+func (r *remoteReadHandle) Close() error {
+	r.readable = nil
+	r.readahead.data = nil
+	return nil
+}
+
+// SetupForCompaction is part of the objstorage.ReadHandle interface.
+func (r *remoteReadHandle) SetupForCompaction() {
+	r.forCompaction = true
+}
+
+// RecordCacheHit is part of the objstorage.ReadHandle interface.
+func (r *remoteReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
+	if !r.forCompaction {
+		r.readahead.state.recordCacheHit(offset, size)
+	}
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/catalog.go

@@ -2,7 +2,7 @@
 // of this source code is governed by a BSD-style license that can be found in
 // the LICENSE file.
 
-package sharedobjcat
+package remoteobjcat
 
 import (
 	"fmt"
@@ -12,23 +12,25 @@ import (
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/remote"
 	"github.com/cockroachdb/pebble/record"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/cockroachdb/pebble/vfs/atomicfs"
 )
 
-// Catalog is used to manage the on-disk shared object catalog.
+// Catalog is used to manage the on-disk remote object catalog.
 //
 // The catalog file is a log of records, where each record is an encoded
-// versionEdit.
+// VersionEdit.
 type Catalog struct {
 	fs      vfs.FS
 	dirname string
 	mu      struct {
 		sync.Mutex
 
-		creatorID CreatorID
+		creatorID objstorage.CreatorID
-		objects   map[base.FileNum]SharedObjectMetadata
+		objects   map[base.DiskFileNum]RemoteObjectMetadata
 
 		marker *atomicfs.Marker
 
@@ -43,44 +45,41 @@ type Catalog struct {
 	}
 }
 
-// CreatorID identifies the DB instance that originally created a shared object.
+// RemoteObjectMetadata encapsulates the data stored in the catalog file for each object.
-// This ID is incorporated in backing object names.
+type RemoteObjectMetadata struct {
-// Must be non-zero.
-type CreatorID uint64
-
-// IsSet returns true if the CreatorID is not zero.
-func (c CreatorID) IsSet() bool { return c != 0 }
-
-func (c CreatorID) String() string { return fmt.Sprintf("%020d", c) }
-
-// SharedObjectMetadata encapsulates the data stored in the catalog file for each object.
-type SharedObjectMetadata struct {
 	// FileNum is the identifier for the object within the context of a single DB
 	// instance.
-	FileNum base.FileNum
+	FileNum base.DiskFileNum
 	// FileType is the type of the object. Only certain FileTypes are possible.
 	FileType base.FileType
 	// CreatorID identifies the DB instance that originally created the object.
-	CreatorID CreatorID
+	CreatorID objstorage.CreatorID
 	// CreatorFileNum is the identifier for the object within the context of the
 	// DB instance that originally created the object.
-	CreatorFileNum base.FileNum
+	CreatorFileNum base.DiskFileNum
+	// CleanupMethod indicates the method for cleaning up unused shared objects.
+	CleanupMethod objstorage.SharedCleanupMethod
+	// Locator identifies a remote.Storage implementation.
+	Locator remote.Locator
+	// CustomObjectName (if it is set) overrides the object name that is normally
+	// derived from the CreatorID and CreatorFileNum.
+	CustomObjectName string
 }
 
 const (
-	catalogFilenameBase = "SHARED-CATALOG"
+	catalogFilenameBase = "REMOTE-OBJ-CATALOG"
-	catalogMarkerName   = "shared-catalog"
+	catalogMarkerName   = "remote-obj-catalog"
 
 	// We create a new file when the size exceeds 1MB (and some other conditions
 	// hold; see record.RotationHelper).
 	rotateFileSize = 1024 * 1024 // 1MB
 )
 
-// CatalogContents contains the shared objects in the catalog.
+// CatalogContents contains the remote objects in the catalog.
 type CatalogContents struct {
 	// CreatorID, if it is set.
-	CreatorID CreatorID
+	CreatorID objstorage.CreatorID
-	Objects   []SharedObjectMetadata
+	Objects   []RemoteObjectMetadata
 }
 
 // Open creates a Catalog and loads any existing catalog file, returning the
@@ -90,7 +89,7 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
 		fs:      fs,
 		dirname: dirname,
 	}
-	c.mu.objects = make(map[base.FileNum]SharedObjectMetadata)
+	c.mu.objects = make(map[base.DiskFileNum]RemoteObjectMetadata)
 
 	var err error
 	c.mu.marker, c.mu.catalogFilename, err = atomicfs.LocateMarker(fs, dirname, catalogMarkerName)
@@ -109,20 +108,20 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
 	}
 	res := CatalogContents{
 		CreatorID: c.mu.creatorID,
-		Objects:   make([]SharedObjectMetadata, 0, len(c.mu.objects)),
+		Objects:   make([]RemoteObjectMetadata, 0, len(c.mu.objects)),
 	}
 	for _, meta := range c.mu.objects {
 		res.Objects = append(res.Objects, meta)
 	}
 	// Sort the objects so the function is deterministic.
 	sort.Slice(res.Objects, func(i, j int) bool {
-		return res.Objects[i].FileNum < res.Objects[j].FileNum
+		return res.Objects[i].FileNum.FileNum() < res.Objects[j].FileNum.FileNum()
 	})
 	return c, res, nil
 }
 
 // SetCreatorID sets the creator ID. If it is already set, it must match.
-func (c *Catalog) SetCreatorID(id CreatorID) error {
+func (c *Catalog) SetCreatorID(id objstorage.CreatorID) error {
 	if !id.IsSet() {
 		return errors.AssertionFailedf("attempt to unset CreatorID")
 	}
@@ -137,9 +136,9 @@ func (c *Catalog) SetCreatorID(id CreatorID) error {
 		return nil
 	}
 
-	ve := versionEdit{CreatorID: id}
+	ve := VersionEdit{CreatorID: id}
 	if err := c.writeToCatalogFileLocked(&ve); err != nil {
-		return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
+		return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
 	}
 	c.mu.creatorID = id
 	return nil
@@ -166,19 +165,19 @@ func (c *Catalog) closeCatalogFile() error {
 
 // Batch is used to perform multiple object additions/deletions at once.
 type Batch struct {
-	ve versionEdit
+	ve VersionEdit
 }
 
 // AddObject adds a new object to the batch.
 //
 // The given FileNum must be new - it must not match that of any object that was
 // ever in the catalog.
-func (b *Batch) AddObject(meta SharedObjectMetadata) {
+func (b *Batch) AddObject(meta RemoteObjectMetadata) {
 	b.ve.NewObjects = append(b.ve.NewObjects, meta)
 }
 
 // DeleteObject adds an object removal to the batch.
-func (b *Batch) DeleteObject(fileNum base.FileNum) {
+func (b *Batch) DeleteObject(fileNum base.DiskFileNum) {
 	b.ve.DeletedObjects = append(b.ve.DeletedObjects, fileNum)
 }
 
@@ -197,11 +196,11 @@ func (b *Batch) IsEmpty() bool {
 func (b *Batch) Copy() Batch {
 	var res Batch
 	if len(b.ve.NewObjects) > 0 {
-		res.ve.NewObjects = make([]SharedObjectMetadata, len(b.ve.NewObjects))
+		res.ve.NewObjects = make([]RemoteObjectMetadata, len(b.ve.NewObjects))
 		copy(res.ve.NewObjects, b.ve.NewObjects)
 	}
 	if len(b.ve.DeletedObjects) > 0 {
-		res.ve.DeletedObjects = make([]base.FileNum, len(b.ve.DeletedObjects))
+		res.ve.DeletedObjects = make([]base.DiskFileNum, len(b.ve.DeletedObjects))
 		copy(res.ve.DeletedObjects, b.ve.DeletedObjects)
 	}
 	return res
@@ -219,29 +218,40 @@ func (c *Catalog) ApplyBatch(b Batch) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	for _, n := range b.ve.DeletedObjects {
+	// Sanity checks.
-		if _, exists := c.mu.objects[n]; !exists {
+	toAdd := make(map[base.DiskFileNum]struct{}, len(b.ve.NewObjects))
-			return errors.AssertionFailedf("deleting non-existent object %s", n)
+	exists := func(n base.DiskFileNum) bool {
+		_, ok := c.mu.objects[n]
+		if !ok {
+			_, ok = toAdd[n]
 		}
+		return ok
 	}
 	for _, meta := range b.ve.NewObjects {
-		if _, exists := c.mu.objects[meta.FileNum]; exists {
+		if exists(meta.FileNum) {
 			return errors.AssertionFailedf("adding existing object %s", meta.FileNum)
 		}
+		toAdd[meta.FileNum] = struct{}{}
+	}
+	for _, n := range b.ve.DeletedObjects {
+		if !exists(n) {
+			return errors.AssertionFailedf("deleting non-existent object %s", n)
+		}
 	}
 
 	if err := c.writeToCatalogFileLocked(&b.ve); err != nil {
-		return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
+		return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
 	}
 
-	// Apply the batch to our current state.
+	// Add new objects before deleting any objects. This allows for cases where
-	for _, n := range b.ve.DeletedObjects {
+	// the same batch adds and deletes an object.
-		delete(c.mu.objects, n)
-	}
 	for _, meta := range b.ve.NewObjects {
 		c.mu.objects[meta.FileNum] = meta
 	}
-	b.Reset()
+	for _, n := range b.ve.DeletedObjects {
+		delete(c.mu.objects, n)
+	}
+
 	return nil
 }
 
@@ -250,7 +260,7 @@ func (c *Catalog) loadFromCatalogFile(filename string) error {
 	f, err := c.fs.Open(catalogPath)
 	if err != nil {
 		return errors.Wrapf(
-			err, "pebble: could not open shared object catalog file %q for DB %q",
+			err, "pebble: could not open remote object catalog file %q for DB %q",
 			errors.Safe(filename), c.dirname,
 		)
 	}
@@ -262,32 +272,26 @@ func (c *Catalog) loadFromCatalogFile(filename string) error {
 			break
 		}
 		if err != nil {
-			return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
+			return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
 				errors.Safe(filename))
 		}
-		var ve versionEdit
+		var ve VersionEdit
-		err = ve.Decode(r)
+		if err := ve.Decode(r); err != nil {
-		if err != nil {
+			return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
-			return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
 				errors.Safe(filename))
 		}
 		// Apply the version edit to the current state.
-		if ve.CreatorID.IsSet() {
+		if err := ve.Apply(&c.mu.creatorID, c.mu.objects); err != nil {
-			c.mu.creatorID = ve.CreatorID
+			return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
-		}
+				errors.Safe(filename))
-		for _, fileNum := range ve.DeletedObjects {
-			delete(c.mu.objects, fileNum)
-		}
-		for _, meta := range ve.NewObjects {
-			c.mu.objects[meta.FileNum] = meta
 		}
 	}
 	return nil
 }
 
-// writeToCatalogFileLocked writes a versionEdit to the catalog file.
+// writeToCatalogFileLocked writes a VersionEdit to the catalog file.
 // Creates a new file if this is the first write.
-func (c *Catalog) writeToCatalogFileLocked(ve *versionEdit) error {
+func (c *Catalog) writeToCatalogFileLocked(ve *VersionEdit) error {
 	c.mu.rotationHelper.AddRecord(int64(len(ve.NewObjects) + len(ve.DeletedObjects)))
 	snapshotSize := int64(len(c.mu.objects))
 
@@ -330,9 +334,10 @@ func (c *Catalog) createNewCatalogFileLocked() (outErr error) {
 	}
 	recWriter := record.NewWriter(file)
 	err = func() error {
-		// Create a versionEdit that gets us from an empty catalog to the current state.
+		// Create a VersionEdit that gets us from an empty catalog to the current state.
-		var ve versionEdit
+		var ve VersionEdit
-		ve.NewObjects = make([]SharedObjectMetadata, 0, len(c.mu.objects))
+		ve.CreatorID = c.mu.creatorID
+		ve.NewObjects = make([]RemoteObjectMetadata, 0, len(c.mu.objects))
 		for _, meta := range c.mu.objects {
 			ve.NewObjects = append(ve.NewObjects, meta)
 		}
@@ -367,7 +372,7 @@ func (c *Catalog) createNewCatalogFileLocked() (outErr error) {
 	return nil
 }
 
-func writeRecord(ve *versionEdit, file vfs.File, recWriter *record.Writer) error {
+func writeRecord(ve *VersionEdit, file vfs.File, recWriter *record.Writer) error {
 	w, err := recWriter.Next()
 	if err != nil {
 		return err

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/version_edit.go

@@ -0,0 +1,254 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package remoteobjcat
+
+import (
+	"bufio"
+	"encoding/binary"
+	"io"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+)
+
+// VersionEdit is a modification to the remote object state which can be encoded
+// into a record.
+//
+// TODO(radu): consider adding creation and deletion time for debugging purposes.
+type VersionEdit struct {
+	NewObjects     []RemoteObjectMetadata
+	DeletedObjects []base.DiskFileNum
+	CreatorID      objstorage.CreatorID
+}
+
+const (
+	// tagNewObject is followed by the FileNum, creator ID, creator FileNum,
+	// cleanup method, optional new object tags, and ending with a 0 byte.
+	tagNewObject = 1
+	// tagDeletedObject is followed by the FileNum.
+	tagDeletedObject = 2
+	// tagCreatorID is followed by the Creator ID for this store. This ID can
+	// never change.
+	tagCreatorID = 3
+	// tagNewObjectLocator is an optional tag inside the tagNewObject payload. It
+	// is followed by the encoded length of the locator string and the string.
+	tagNewObjectLocator = 4
+	// tagNewObjectCustomName is an optional tag inside the tagNewObject payload.
+	// It is followed by the encoded length of the custom object name string
+	// followed by the string.
+	tagNewObjectCustomName = 5
+)
+
+// Object type values. We don't want to encode FileType directly because it is
+// more general (and we want freedom to change it in the future).
+const (
+	objTypeTable = 1
+)
+
+func objTypeToFileType(objType uint64) (base.FileType, error) {
+	switch objType {
+	case objTypeTable:
+		return base.FileTypeTable, nil
+	default:
+		return 0, errors.Newf("unknown object type %d", objType)
+	}
+}
+
+func fileTypeToObjType(fileType base.FileType) (uint64, error) {
+	switch fileType {
+	case base.FileTypeTable:
+		return objTypeTable, nil
+
+	default:
+		return 0, errors.Newf("unknown object type for file type %d", fileType)
+	}
+}
+
+// Encode encodes an edit to the specified writer.
+func (v *VersionEdit) Encode(w io.Writer) error {
+	buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*10+len(v.DeletedObjects)*2+2))
+	for _, meta := range v.NewObjects {
+		objType, err := fileTypeToObjType(meta.FileType)
+		if err != nil {
+			return err
+		}
+		buf = binary.AppendUvarint(buf, uint64(tagNewObject))
+		buf = binary.AppendUvarint(buf, uint64(meta.FileNum.FileNum()))
+		buf = binary.AppendUvarint(buf, objType)
+		buf = binary.AppendUvarint(buf, uint64(meta.CreatorID))
+		buf = binary.AppendUvarint(buf, uint64(meta.CreatorFileNum.FileNum()))
+		buf = binary.AppendUvarint(buf, uint64(meta.CleanupMethod))
+		if meta.Locator != "" {
+			buf = binary.AppendUvarint(buf, uint64(tagNewObjectLocator))
+			buf = encodeString(buf, string(meta.Locator))
+		}
+		if meta.CustomObjectName != "" {
+			buf = binary.AppendUvarint(buf, uint64(tagNewObjectCustomName))
+			buf = encodeString(buf, meta.CustomObjectName)
+		}
+		// Append 0 as the terminator for optional new object tags.
+		buf = binary.AppendUvarint(buf, 0)
+	}
+
+	for _, dfn := range v.DeletedObjects {
+		buf = binary.AppendUvarint(buf, uint64(tagDeletedObject))
+		buf = binary.AppendUvarint(buf, uint64(dfn.FileNum()))
+	}
+	if v.CreatorID.IsSet() {
+		buf = binary.AppendUvarint(buf, uint64(tagCreatorID))
+		buf = binary.AppendUvarint(buf, uint64(v.CreatorID))
+	}
+	_, err := w.Write(buf)
+	return err
+}
+
+// Decode decodes an edit from the specified reader.
+func (v *VersionEdit) Decode(r io.Reader) error {
+	br, ok := r.(io.ByteReader)
+	if !ok {
+		br = bufio.NewReader(r)
+	}
+	for {
+		tag, err := binary.ReadUvarint(br)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		err = nil
+		switch tag {
+		case tagNewObject:
+			var fileNum, creatorID, creatorFileNum, cleanupMethod uint64
+			var locator, customName string
+			var fileType base.FileType
+			fileNum, err = binary.ReadUvarint(br)
+			if err == nil {
+				var objType uint64
+				objType, err = binary.ReadUvarint(br)
+				if err == nil {
+					fileType, err = objTypeToFileType(objType)
+				}
+			}
+			if err == nil {
+				creatorID, err = binary.ReadUvarint(br)
+			}
+			if err == nil {
+				creatorFileNum, err = binary.ReadUvarint(br)
+			}
+			if err == nil {
+				cleanupMethod, err = binary.ReadUvarint(br)
+			}
+			for err == nil {
+				var optionalTag uint64
+				optionalTag, err = binary.ReadUvarint(br)
+				if err != nil || optionalTag == 0 {
+					break
+				}
+
+				switch optionalTag {
+				case tagNewObjectLocator:
+					locator, err = decodeString(br)
+
+				case tagNewObjectCustomName:
+					customName, err = decodeString(br)
+
+				default:
+					err = errors.Newf("unknown newObject tag %d", optionalTag)
+				}
+			}
+
+			if err == nil {
+				v.NewObjects = append(v.NewObjects, RemoteObjectMetadata{
+					FileNum:          base.FileNum(fileNum).DiskFileNum(),
+					FileType:         fileType,
+					CreatorID:        objstorage.CreatorID(creatorID),
+					CreatorFileNum:   base.FileNum(creatorFileNum).DiskFileNum(),
+					CleanupMethod:    objstorage.SharedCleanupMethod(cleanupMethod),
+					Locator:          remote.Locator(locator),
+					CustomObjectName: customName,
+				})
+			}
+
+		case tagDeletedObject:
+			var fileNum uint64
+			fileNum, err = binary.ReadUvarint(br)
+			if err == nil {
+				v.DeletedObjects = append(v.DeletedObjects, base.FileNum(fileNum).DiskFileNum())
+			}
+
+		case tagCreatorID:
+			var id uint64
+			id, err = binary.ReadUvarint(br)
+			if err == nil {
+				v.CreatorID = objstorage.CreatorID(id)
+			}
+
+		default:
+			err = errors.Newf("unknown tag %d", tag)
+		}
+
+		if err != nil {
+			if err == io.EOF {
+				return errCorruptCatalog
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+func encodeString(buf []byte, s string) []byte {
+	buf = binary.AppendUvarint(buf, uint64(len(s)))
+	buf = append(buf, []byte(s)...)
+	return buf
+}
+
+func decodeString(br io.ByteReader) (string, error) {
+	length, err := binary.ReadUvarint(br)
+	if err != nil || length == 0 {
+		return "", err
+	}
+	buf := make([]byte, length)
+	for i := range buf {
+		buf[i], err = br.ReadByte()
+		if err != nil {
+			return "", err
+		}
+	}
+	return string(buf), nil
+}
+
+var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt remote object catalog")
+
+// Apply the version edit to a creator ID and a map of objects.
+func (v *VersionEdit) Apply(
+	creatorID *objstorage.CreatorID, objects map[base.DiskFileNum]RemoteObjectMetadata,
+) error {
+	if v.CreatorID.IsSet() {
+		*creatorID = v.CreatorID
+	}
+	for _, meta := range v.NewObjects {
+		if invariants.Enabled {
+			if _, exists := objects[meta.FileNum]; exists {
+				return errors.AssertionFailedf("version edit adds existing object %s", meta.FileNum)
+			}
+		}
+		objects[meta.FileNum] = meta
+	}
+	for _, fileNum := range v.DeletedObjects {
+		if invariants.Enabled {
+			if _, exists := objects[fileNum]; !exists {
+				return errors.AssertionFailedf("version edit deletes non-existent object %s", fileNum)
+			}
+		}
+		delete(objects, fileNum)
+	}
+	return nil
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_writable.go

@@ -0,0 +1,65 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"io"
+
+	"github.com/cockroachdb/pebble/objstorage"
+)
+
+// NewRemoteWritable creates an objstorage.Writable out of an io.WriteCloser.
+func NewRemoteWritable(obj io.WriteCloser) objstorage.Writable {
+	return &sharedWritable{storageWriter: obj}
+}
+
+// sharedWritable is a very simple implementation of Writable on top of the
+// WriteCloser returned by remote.Storage.CreateObject.
+type sharedWritable struct {
+	// Either both p and meta must be unset / zero values, or both must be set.
+	// The case where both are unset is true only in tests.
+	p             *provider
+	meta          objstorage.ObjectMetadata
+	storageWriter io.WriteCloser
+}
+
+var _ objstorage.Writable = (*sharedWritable)(nil)
+
+// Write is part of the Writable interface.
+func (w *sharedWritable) Write(p []byte) error {
+	_, err := w.storageWriter.Write(p)
+	return err
+}
+
+// Finish is part of the Writable interface.
+func (w *sharedWritable) Finish() error {
+	err := w.storageWriter.Close()
+	w.storageWriter = nil
+	if err != nil {
+		w.Abort()
+		return err
+	}
+
+	// Create the marker object.
+	if w.p != nil {
+		if err := w.p.sharedCreateRef(w.meta); err != nil {
+			w.Abort()
+			return err
+		}
+	}
+	return nil
+}
+
+// Abort is part of the Writable interface.
+func (w *sharedWritable) Abort() {
+	if w.storageWriter != nil {
+		_ = w.storageWriter.Close()
+		w.storageWriter = nil
+	}
+	if w.p != nil {
+		w.p.removeMetadata(w.meta.DiskFileNum)
+	}
+	// TODO(radu): delete the object if it was created.
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache/shared_cache.go

@@ -0,0 +1,900 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package sharedcache
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math/bits"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+	"github.com/cockroachdb/pebble/vfs"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Exported to enable exporting from package pebble to enable
+// exporting metrics with below buckets in CRDB.
+var (
+	IOBuckets           = prometheus.ExponentialBucketsRange(float64(time.Millisecond*1), float64(10*time.Second), 50)
+	ChannelWriteBuckets = prometheus.ExponentialBucketsRange(float64(time.Microsecond*1), float64(10*time.Second), 50)
+)
+
+// Cache is a persistent cache backed by a local filesystem. It is intended
+// to cache data that is in slower shared storage (e.g. S3), hence the
+// package name 'sharedcache'.
+type Cache struct {
+	shards       []shard
+	writeWorkers writeWorkers
+
+	bm                blockMath
+	shardingBlockSize int64
+
+	logger  base.Logger
+	metrics internalMetrics
+}
+
+// Metrics is a struct containing metrics exported by the secondary cache.
+// TODO(josh): Reconsider the set of metrics exported by the secondary cache
+// before we release the secondary cache to users. We choose to export many metrics
+// right now, so we learn a lot from the benchmarking we are doing over the 23.2
+// cycle.
+type Metrics struct {
+	// The number of sstable bytes stored in the cache.
+	Size int64
+	// The count of cache blocks in the cache (not sstable blocks).
+	Count int64
+
+	// The number of calls to ReadAt.
+	TotalReads int64
+	// The number of calls to ReadAt that require reading data from 2+ shards.
+	MultiShardReads int64
+	// The number of calls to ReadAt that require reading data from 2+ cache blocks.
+	MultiBlockReads int64
+	// The number of calls to ReadAt where all data returned was read from the cache.
+	ReadsWithFullHit int64
+	// The number of calls to ReadAt where some data returned was read from the cache.
+	ReadsWithPartialHit int64
+	// The number of calls to ReadAt where no data returned was read from the cache.
+	ReadsWithNoHit int64
+
+	// The number of times a cache block was evicted from the cache.
+	Evictions int64
+	// The number of times writing a cache block to the cache failed.
+	WriteBackFailures int64
+
+	// The latency of calls to get some data from the cache.
+	GetLatency prometheus.Histogram
+	// The latency of reads of a single cache block from disk.
+	DiskReadLatency prometheus.Histogram
+	// The latency of writing data to write back to the cache to a channel.
+	// Generally should be low, but if the channel is full, could be high.
+	QueuePutLatency prometheus.Histogram
+	// The latency of calls to put some data read from block storage into the cache.
+	PutLatency prometheus.Histogram
+	// The latency of writes of a single cache block to disk.
+	DiskWriteLatency prometheus.Histogram
+}
+
+// See docs at Metrics.
+type internalMetrics struct {
+	count atomic.Int64
+
+	totalReads          atomic.Int64
+	multiShardReads     atomic.Int64
+	multiBlockReads     atomic.Int64
+	readsWithFullHit    atomic.Int64
+	readsWithPartialHit atomic.Int64
+	readsWithNoHit      atomic.Int64
+
+	evictions         atomic.Int64
+	writeBackFailures atomic.Int64
+
+	getLatency       prometheus.Histogram
+	diskReadLatency  prometheus.Histogram
+	queuePutLatency  prometheus.Histogram
+	putLatency       prometheus.Histogram
+	diskWriteLatency prometheus.Histogram
+}
+
+const (
+	// writeWorkersPerShard is used to establish the number of worker goroutines
+	// that perform writes to the cache.
+	writeWorkersPerShard = 4
+	// writeTaskPerWorker is used to establish how many tasks can be queued up
+	// until we have to block.
+	writeTasksPerWorker = 4
+)
+
+// Open opens a cache. If there is no existing cache at fsDir, a new one
+// is created.
+func Open(
+	fs vfs.FS,
+	logger base.Logger,
+	fsDir string,
+	blockSize int,
+	// shardingBlockSize is the size of a shard block. The cache is split into contiguous
+	// shardingBlockSize units. The units are distributed across multiple independent shards
+	// of the cache, via a hash(offset) modulo num shards operation. The cache replacement
+	// policies operate at the level of shard, not whole cache. This is done to reduce lock
+	// contention.
+	shardingBlockSize int64,
+	sizeBytes int64,
+	numShards int,
+) (*Cache, error) {
+	if minSize := shardingBlockSize * int64(numShards); sizeBytes < minSize {
+		// Up the size so that we have one block per shard. In practice, this should
+		// only happen in tests.
+		sizeBytes = minSize
+	}
+
+	c := &Cache{
+		logger:            logger,
+		bm:                makeBlockMath(blockSize),
+		shardingBlockSize: shardingBlockSize,
+	}
+	c.shards = make([]shard, numShards)
+	blocksPerShard := sizeBytes / int64(numShards) / int64(blockSize)
+	for i := range c.shards {
+		if err := c.shards[i].init(c, fs, fsDir, i, blocksPerShard, blockSize, shardingBlockSize); err != nil {
+			return nil, err
+		}
+	}
+
+	c.writeWorkers.Start(c, numShards*writeWorkersPerShard)
+
+	c.metrics.getLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
+	c.metrics.diskReadLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
+	c.metrics.putLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
+	c.metrics.diskWriteLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
+
+	// Measures a channel write, so lower min.
+	c.metrics.queuePutLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: ChannelWriteBuckets})
+
+	return c, nil
+}
+
+// Close closes the cache. Methods such as ReadAt should not be called after Close is
+// called.
+func (c *Cache) Close() error {
+	c.writeWorkers.Stop()
+
+	var retErr error
+	for i := range c.shards {
+		if err := c.shards[i].close(); err != nil && retErr == nil {
+			retErr = err
+		}
+	}
+	c.shards = nil
+	return retErr
+}
+
+// Metrics return metrics for the cache. Callers should not mutate
+// the returned histograms, which are pointer types.
+func (c *Cache) Metrics() Metrics {
+	return Metrics{
+		Count:               c.metrics.count.Load(),
+		Size:                c.metrics.count.Load() * int64(c.bm.BlockSize()),
+		TotalReads:          c.metrics.totalReads.Load(),
+		MultiShardReads:     c.metrics.multiShardReads.Load(),
+		MultiBlockReads:     c.metrics.multiBlockReads.Load(),
+		ReadsWithFullHit:    c.metrics.readsWithFullHit.Load(),
+		ReadsWithPartialHit: c.metrics.readsWithPartialHit.Load(),
+		ReadsWithNoHit:      c.metrics.readsWithNoHit.Load(),
+		Evictions:           c.metrics.evictions.Load(),
+		WriteBackFailures:   c.metrics.writeBackFailures.Load(),
+		GetLatency:          c.metrics.getLatency,
+		DiskReadLatency:     c.metrics.diskReadLatency,
+		QueuePutLatency:     c.metrics.queuePutLatency,
+		PutLatency:          c.metrics.putLatency,
+		DiskWriteLatency:    c.metrics.diskWriteLatency,
+	}
+}
+
+// ReadFlags contains options for Cache.ReadAt.
+type ReadFlags struct {
+	// ReadOnly instructs ReadAt to not write any new data into the cache; it is
+	// used when the data is unlikely to be used again.
+	ReadOnly bool
+}
+
+// ReadAt performs a read form an object, attempting to use cached data when
+// possible.
+func (c *Cache) ReadAt(
+	ctx context.Context,
+	fileNum base.DiskFileNum,
+	p []byte,
+	ofs int64,
+	objReader remote.ObjectReader,
+	objSize int64,
+	flags ReadFlags,
+) error {
+	c.metrics.totalReads.Add(1)
+	if ofs >= objSize {
+		if invariants.Enabled {
+			panic(fmt.Sprintf("invalid ReadAt offset %v %v", ofs, objSize))
+		}
+		return io.EOF
+	}
+	// TODO(radu): for compaction reads, we may not want to read from the cache at
+	// all.
+	{
+		start := time.Now()
+		n, err := c.get(fileNum, p, ofs)
+		c.metrics.getLatency.Observe(float64(time.Since(start)))
+		if err != nil {
+			return err
+		}
+		if n == len(p) {
+			// Everything was in cache!
+			c.metrics.readsWithFullHit.Add(1)
+			return nil
+		}
+		if n == 0 {
+			c.metrics.readsWithNoHit.Add(1)
+		} else {
+			c.metrics.readsWithPartialHit.Add(1)
+		}
+
+		// Note this. The below code does not need the original ofs, as with the earlier
+		// reading from the cache done, the relevant offset is ofs + int64(n). Same with p.
+		ofs += int64(n)
+		p = p[n:]
+
+		if invariants.Enabled {
+			if n != 0 && c.bm.Remainder(ofs) != 0 {
+				panic(fmt.Sprintf("after non-zero read from cache, ofs is not block-aligned: %v %v", ofs, n))
+			}
+		}
+	}
+
+	if flags.ReadOnly {
+		return objReader.ReadAt(ctx, p, ofs)
+	}
+
+	// We must do reads with offset & size that are multiples of the block size. Else
+	// later cache hits may return incorrect zeroed results from the cache.
+	firstBlockInd := c.bm.Block(ofs)
+	adjustedOfs := c.bm.BlockOffset(firstBlockInd)
+
+	// Take the length of what is left to read plus the length of the adjustment of
+	// the offset plus the size of a block minus one and divide by the size of a block
+	// to get the number of blocks to read from the object.
+	sizeOfOffAdjustment := int(ofs - adjustedOfs)
+	adjustedLen := int(c.bm.RoundUp(int64(len(p) + sizeOfOffAdjustment)))
+	adjustedP := make([]byte, adjustedLen)
+
+	// Read the rest from the object. We may need to cap the length to avoid past EOF reads.
+	eofCap := int64(adjustedLen)
+	if adjustedOfs+eofCap > objSize {
+		eofCap = objSize - adjustedOfs
+	}
+	if err := objReader.ReadAt(ctx, adjustedP[:eofCap], adjustedOfs); err != nil {
+		return err
+	}
+	copy(p, adjustedP[sizeOfOffAdjustment:])
+
+	start := time.Now()
+	c.writeWorkers.QueueWrite(fileNum, adjustedP, adjustedOfs)
+	c.metrics.queuePutLatency.Observe(float64(time.Since(start)))
+
+	return nil
+}
+
+// get attempts to read the requested data from the cache, if it is already
+// there.
+//
+// If all data is available, returns n = len(p).
+//
+// If data is partially available, a prefix of the data is read; returns n < len(p)
+// and no error. If no prefix is available, returns n = 0 and no error.
+func (c *Cache) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ error) {
+	// The data extent might cross shard boundaries, hence the loop. In the hot
+	// path, max two iterations of this loop will be executed, since reads are sized
+	// in units of sstable block size.
+	var multiShard bool
+	for {
+		shard := c.getShard(fileNum, ofs+int64(n))
+		cappedLen := len(p[n:])
+		if toBoundary := int(c.shardingBlockSize - ((ofs + int64(n)) % c.shardingBlockSize)); cappedLen > toBoundary {
+			cappedLen = toBoundary
+		}
+		numRead, err := shard.get(fileNum, p[n:n+cappedLen], ofs+int64(n))
+		if err != nil {
+			return n, err
+		}
+		n += numRead
+		if numRead < cappedLen {
+			// We only read a prefix from this shard.
+			return n, nil
+		}
+		if n == len(p) {
+			// We are done.
+			return n, nil
+		}
+		// Data extent crosses shard boundary, continue with next shard.
+		if !multiShard {
+			c.metrics.multiShardReads.Add(1)
+			multiShard = true
+		}
+	}
+}
+
+// set attempts to write the requested data to the cache. Both ofs & len(p) must
+// be multiples of the block size.
+//
+// If all of p is not written to the shard, set returns a non-nil error.
+func (c *Cache) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
+	if invariants.Enabled {
+		if c.bm.Remainder(ofs) != 0 || c.bm.Remainder(int64(len(p))) != 0 {
+			panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p)))
+		}
+	}
+
+	// The data extent might cross shard boundaries, hence the loop. In the hot
+	// path, max two iterations of this loop will be executed, since reads are sized
+	// in units of sstable block size.
+	n := 0
+	for {
+		shard := c.getShard(fileNum, ofs+int64(n))
+		cappedLen := len(p[n:])
+		if toBoundary := int(c.shardingBlockSize - ((ofs + int64(n)) % c.shardingBlockSize)); cappedLen > toBoundary {
+			cappedLen = toBoundary
+		}
+		err := shard.set(fileNum, p[n:n+cappedLen], ofs+int64(n))
+		if err != nil {
+			return err
+		}
+		// set returns an error if cappedLen bytes aren't written to the shard.
+		n += cappedLen
+		if n == len(p) {
+			// We are done.
+			return nil
+		}
+		// Data extent crosses shard boundary, continue with next shard.
+	}
+}
+
+func (c *Cache) getShard(fileNum base.DiskFileNum, ofs int64) *shard {
+	const prime64 = 1099511628211
+	hash := uint64(fileNum.FileNum())*prime64 + uint64(ofs/c.shardingBlockSize)
+	// TODO(josh): Instance change ops are often run in production. Such an operation
+	// updates len(c.shards); see openSharedCache. As a result, the behavior of this
+	// function changes, and the cache empties out at restart time. We may want a better
+	// story here eventually.
+	return &c.shards[hash%uint64(len(c.shards))]
+}
+
+type shard struct {
+	cache             *Cache
+	file              vfs.File
+	sizeInBlocks      int64
+	bm                blockMath
+	shardingBlockSize int64
+	mu                struct {
+		sync.Mutex
+		// TODO(josh): None of these datastructures are space-efficient.
+		// Focusing on correctness to start.
+		where  whereMap
+		blocks []cacheBlockState
+		// Head of LRU list (doubly-linked circular).
+		lruHead cacheBlockIndex
+		// Head of free list (singly-linked chain).
+		freeHead cacheBlockIndex
+	}
+}
+
+type cacheBlockState struct {
+	lock    lockState
+	logical logicalBlockID
+
+	// next is the next block in the LRU or free list (or invalidBlockIndex if it
+	// is the last block in the free list).
+	next cacheBlockIndex
+
+	// prev is the previous block in the LRU list. It is not used when the block
+	// is in the free list.
+	prev cacheBlockIndex
+}
+
+// Maps a logical block in an SST to an index of the cache block with the
+// file contents (to the "cache block index").
+type whereMap map[logicalBlockID]cacheBlockIndex
+
+type logicalBlockID struct {
+	filenum       base.DiskFileNum
+	cacheBlockIdx cacheBlockIndex
+}
+
+type lockState int64
+
+const (
+	unlocked lockState = 0
+	// >0 lockState tracks the number of distinct readers of some cache block / logical block
+	// which is in the secondary cache. It is used to ensure that a cache block is not evicted
+	// and overwritten, while there are active readers.
+	readLockTakenInc = 1
+	// -1 lockState indicates that some cache block is currently being populated with data from
+	// blob storage. It is used to ensure that a cache block is not read or evicted again, while
+	// it is being populated.
+	writeLockTaken = -1
+)
+
+func (s *shard) init(
+	cache *Cache,
+	fs vfs.FS,
+	fsDir string,
+	shardIdx int,
+	sizeInBlocks int64,
+	blockSize int,
+	shardingBlockSize int64,
+) error {
+	*s = shard{
+		cache:        cache,
+		sizeInBlocks: sizeInBlocks,
+	}
+	if blockSize < 1024 || shardingBlockSize%int64(blockSize) != 0 {
+		return errors.Newf("invalid block size %d (must divide %d)", blockSize, shardingBlockSize)
+	}
+	s.bm = makeBlockMath(blockSize)
+	s.shardingBlockSize = shardingBlockSize
+	file, err := fs.OpenReadWrite(fs.PathJoin(fsDir, fmt.Sprintf("SHARED-CACHE-%03d", shardIdx)))
+	if err != nil {
+		return err
+	}
+	// TODO(radu): truncate file if necessary (especially important if we restart
+	// with more shards).
+	if err := file.Preallocate(0, int64(blockSize)*sizeInBlocks); err != nil {
+		return err
+	}
+	s.file = file
+
+	// TODO(josh): Right now, the secondary cache is not persistent. All existing
+	// cache contents will be over-written, since all metadata is only stored in
+	// memory.
+	s.mu.where = make(whereMap)
+	s.mu.blocks = make([]cacheBlockState, sizeInBlocks)
+	s.mu.lruHead = invalidBlockIndex
+	s.mu.freeHead = invalidBlockIndex
+	for i := range s.mu.blocks {
+		s.freePush(cacheBlockIndex(i))
+	}
+
+	return nil
+}
+
+func (s *shard) close() error {
+	defer func() {
+		s.file = nil
+	}()
+	return s.file.Close()
+}
+
+// freePush pushes a block to the front of the free list.
+func (s *shard) freePush(index cacheBlockIndex) {
+	s.mu.blocks[index].next = s.mu.freeHead
+	s.mu.freeHead = index
+}
+
+// freePop removes the block from the front of the free list. Must not be called
+// if the list is empty (i.e. freeHead = invalidBlockIndex).
+func (s *shard) freePop() cacheBlockIndex {
+	index := s.mu.freeHead
+	s.mu.freeHead = s.mu.blocks[index].next
+	return index
+}
+
+// lruInsertFront inserts a block at the front of the LRU list.
+func (s *shard) lruInsertFront(index cacheBlockIndex) {
+	b := &s.mu.blocks[index]
+	if s.mu.lruHead == invalidBlockIndex {
+		b.next = index
+		b.prev = index
+	} else {
+		b.next = s.mu.lruHead
+		h := &s.mu.blocks[s.mu.lruHead]
+		b.prev = h.prev
+		s.mu.blocks[h.prev].next = index
+		h.prev = index
+	}
+	s.mu.lruHead = index
+}
+
+func (s *shard) lruNext(index cacheBlockIndex) cacheBlockIndex {
+	return s.mu.blocks[index].next
+}
+
+func (s *shard) lruPrev(index cacheBlockIndex) cacheBlockIndex {
+	return s.mu.blocks[index].prev
+}
+
+// lruUnlink removes a block from the LRU list.
+func (s *shard) lruUnlink(index cacheBlockIndex) {
+	b := &s.mu.blocks[index]
+	if b.next == index {
+		s.mu.lruHead = invalidBlockIndex
+	} else {
+		s.mu.blocks[b.prev].next = b.next
+		s.mu.blocks[b.next].prev = b.prev
+		if s.mu.lruHead == index {
+			s.mu.lruHead = b.next
+		}
+	}
+	b.next, b.prev = invalidBlockIndex, invalidBlockIndex
+}
+
+// get attempts to read the requested data from the shard. The data must not
+// cross a shard boundary.
+//
+// If all data is available, returns n = len(p).
+//
+// If data is partially available, a prefix of the data is read; returns n < len(p)
+// and no error. If no prefix is available, returns n = 0 and no error.
+//
+// TODO(josh): Today, if there are two cache blocks needed to satisfy a read, and the
+// first block is not in the cache and the second one is, we will read both from
+// blob storage. We should fix this. This is not an unlikely scenario if we are doing
+// a reverse scan, since those iterate over sstable blocks in reverse order and due to
+// cache block aligned reads will have read the suffix of the sstable block that will
+// be needed next.
+func (s *shard) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ error) {
+	if invariants.Enabled {
+		if ofs/s.shardingBlockSize != (ofs+int64(len(p))-1)/s.shardingBlockSize {
+			panic(fmt.Sprintf("get crosses shard boundary: %v %v", ofs, len(p)))
+		}
+		s.assertShardStateIsConsistent()
+	}
+
+	// The data extent might cross cache block boundaries, hence the loop. In the hot
+	// path, max two iterations of this loop will be executed, since reads are sized
+	// in units of sstable block size.
+	var multiBlock bool
+	for {
+		k := logicalBlockID{
+			filenum:       fileNum,
+			cacheBlockIdx: s.bm.Block(ofs + int64(n)),
+		}
+		s.mu.Lock()
+		cacheBlockIdx, ok := s.mu.where[k]
+		// TODO(josh): Multiple reads within the same few milliseconds (anything that is smaller
+		// than blob storage read latency) that miss on the same logical block ID will not necessarily
+		// be rare. We may want to do only one read, with the later readers blocking on the first read
+		// completing. This could be implemented either here or in the primary block cache. See
+		// https://github.com/cockroachdb/pebble/pull/2586 for additional discussion.
+		if !ok {
+			s.mu.Unlock()
+			return n, nil
+		}
+		if s.mu.blocks[cacheBlockIdx].lock == writeLockTaken {
+			// In practice, if we have two reads of the same SST block in close succession, we
+			// would expect the second to hit in the in-memory block cache. So it's not worth
+			// optimizing this case here.
+			s.mu.Unlock()
+			return n, nil
+		}
+		s.mu.blocks[cacheBlockIdx].lock += readLockTakenInc
+		// Move to front of the LRU list.
+		s.lruUnlink(cacheBlockIdx)
+		s.lruInsertFront(cacheBlockIdx)
+		s.mu.Unlock()
+
+		readAt := s.bm.BlockOffset(cacheBlockIdx)
+		readSize := s.bm.BlockSize()
+		if n == 0 { // if first read
+			rem := s.bm.Remainder(ofs)
+			readAt += rem
+			readSize -= int(rem)
+		}
+
+		if len(p[n:]) <= readSize {
+			start := time.Now()
+			numRead, err := s.file.ReadAt(p[n:], readAt)
+			s.cache.metrics.diskReadLatency.Observe(float64(time.Since(start)))
+			s.dropReadLock(cacheBlockIdx)
+			return n + numRead, err
+		}
+		start := time.Now()
+		numRead, err := s.file.ReadAt(p[n:n+readSize], readAt)
+		s.cache.metrics.diskReadLatency.Observe(float64(time.Since(start)))
+		s.dropReadLock(cacheBlockIdx)
+		if err != nil {
+			return 0, err
+		}
+
+		// Note that numRead == readSize, since we checked for an error above.
+		n += numRead
+
+		if !multiBlock {
+			s.cache.metrics.multiBlockReads.Add(1)
+			multiBlock = true
+		}
+	}
+}
+
+// set attempts to write the requested data to the shard. The data must not
+// cross a shard boundary, and both ofs & len(p) must be multiples of the
+// block size.
+//
+// If all of p is not written to the shard, set returns a non-nil error.
+func (s *shard) set(fileNum base.DiskFileNum, p []byte, ofs int64) error {
+	if invariants.Enabled {
+		if ofs/s.shardingBlockSize != (ofs+int64(len(p))-1)/s.shardingBlockSize {
+			panic(fmt.Sprintf("set crosses shard boundary: %v %v", ofs, len(p)))
+		}
+		if s.bm.Remainder(ofs) != 0 || s.bm.Remainder(int64(len(p))) != 0 {
+			panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p)))
+		}
+		s.assertShardStateIsConsistent()
+	}
+
+	// The data extent might cross cache block boundaries, hence the loop. In the hot
+	// path, max two iterations of this loop will be executed, since reads are sized
+	// in units of sstable block size.
+	n := 0
+	for {
+		if n == len(p) {
+			return nil
+		}
+		if invariants.Enabled {
+			if n > len(p) {
+				panic(fmt.Sprintf("set with n greater than len(p): %v %v", n, len(p)))
+			}
+		}
+
+		// If the logical block is already in the cache, we should skip doing a set.
+		k := logicalBlockID{
+			filenum:       fileNum,
+			cacheBlockIdx: s.bm.Block(ofs + int64(n)),
+		}
+		s.mu.Lock()
+		if _, ok := s.mu.where[k]; ok {
+			s.mu.Unlock()
+			n += s.bm.BlockSize()
+			continue
+		}
+
+		var cacheBlockIdx cacheBlockIndex
+		if s.mu.freeHead == invalidBlockIndex {
+			if invariants.Enabled && s.mu.lruHead == invalidBlockIndex {
+				panic("both LRU and free lists empty")
+			}
+
+			// Find the last element in the LRU list which is not locked.
+			for idx := s.lruPrev(s.mu.lruHead); ; idx = s.lruPrev(idx) {
+				if lock := s.mu.blocks[idx].lock; lock == unlocked {
+					cacheBlockIdx = idx
+					break
+				}
+				if idx == s.mu.lruHead {
+					// No unlocked block to evict.
+					//
+					// TODO(josh): We may want to block until a block frees up, instead of returning
+					// an error here. But I think we can do that later on, e.g. after running some production
+					// experiments.
+					s.mu.Unlock()
+					return errors.New("no block to evict so skipping write to cache")
+				}
+			}
+			s.cache.metrics.evictions.Add(1)
+			s.lruUnlink(cacheBlockIdx)
+			delete(s.mu.where, s.mu.blocks[cacheBlockIdx].logical)
+		} else {
+			s.cache.metrics.count.Add(1)
+			cacheBlockIdx = s.freePop()
+		}
+
+		s.lruInsertFront(cacheBlockIdx)
+		s.mu.where[k] = cacheBlockIdx
+		s.mu.blocks[cacheBlockIdx].logical = k
+		s.mu.blocks[cacheBlockIdx].lock = writeLockTaken
+		s.mu.Unlock()
+
+		writeAt := s.bm.BlockOffset(cacheBlockIdx)
+
+		writeSize := s.bm.BlockSize()
+		if len(p[n:]) <= writeSize {
+			writeSize = len(p[n:])
+		}
+
+		start := time.Now()
+		_, err := s.file.WriteAt(p[n:n+writeSize], writeAt)
+		s.cache.metrics.diskWriteLatency.Observe(float64(time.Since(start)))
+		if err != nil {
+			// Free the block.
+			s.mu.Lock()
+			defer s.mu.Unlock()
+
+			delete(s.mu.where, k)
+			s.lruUnlink(cacheBlockIdx)
+			s.freePush(cacheBlockIdx)
+			return err
+		}
+		s.dropWriteLock(cacheBlockIdx)
+		n += writeSize
+	}
+}
+
+// Doesn't inline currently. This might be okay, but something to keep in mind.
+func (s *shard) dropReadLock(cacheBlockInd cacheBlockIndex) {
+	s.mu.Lock()
+	s.mu.blocks[cacheBlockInd].lock -= readLockTakenInc
+	if invariants.Enabled && s.mu.blocks[cacheBlockInd].lock < 0 {
+		panic(fmt.Sprintf("unexpected lock state %v in dropReadLock", s.mu.blocks[cacheBlockInd].lock))
+	}
+	s.mu.Unlock()
+}
+
+// Doesn't inline currently. This might be okay, but something to keep in mind.
+func (s *shard) dropWriteLock(cacheBlockInd cacheBlockIndex) {
+	s.mu.Lock()
+	if invariants.Enabled && s.mu.blocks[cacheBlockInd].lock != writeLockTaken {
+		panic(fmt.Sprintf("unexpected lock state %v in dropWriteLock", s.mu.blocks[cacheBlockInd].lock))
+	}
+	s.mu.blocks[cacheBlockInd].lock = unlocked
+	s.mu.Unlock()
+}
+
+func (s *shard) assertShardStateIsConsistent() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	lruLen := 0
+	if s.mu.lruHead != invalidBlockIndex {
+		for b := s.mu.lruHead; ; {
+			lruLen++
+			if idx, ok := s.mu.where[s.mu.blocks[b].logical]; !ok || idx != b {
+				panic("block in LRU list with no entry in where map")
+			}
+			b = s.lruNext(b)
+			if b == s.mu.lruHead {
+				break
+			}
+		}
+	}
+	if lruLen != len(s.mu.where) {
+		panic(fmt.Sprintf("lru list len is %d but where map has %d entries", lruLen, len(s.mu.where)))
+	}
+	freeLen := 0
+	for n := s.mu.freeHead; n != invalidBlockIndex; n = s.mu.blocks[n].next {
+		freeLen++
+	}
+
+	if lruLen+freeLen != int(s.sizeInBlocks) {
+		panic(fmt.Sprintf("%d lru blocks and %d free blocks don't add up to %d", lruLen, freeLen, s.sizeInBlocks))
+	}
+	for i := range s.mu.blocks {
+		if state := s.mu.blocks[i].lock; state < writeLockTaken {
+			panic(fmt.Sprintf("lock state %v is not allowed", state))
+		}
+	}
+}
+
+// cacheBlockIndex is the index of a blockSize-aligned cache block.
+type cacheBlockIndex int64
+
+// invalidBlockIndex is used for the head of a list when the list is empty.
+const invalidBlockIndex cacheBlockIndex = -1
+
+// blockMath is a helper type for performing conversions between offsets and
+// block indexes.
+type blockMath struct {
+	blockSizeBits int8
+}
+
+func makeBlockMath(blockSize int) blockMath {
+	bm := blockMath{
+		blockSizeBits: int8(bits.Len64(uint64(blockSize)) - 1),
+	}
+	if blockSize != (1 << bm.blockSizeBits) {
+		panic(fmt.Sprintf("blockSize %d is not a power of 2", blockSize))
+	}
+	return bm
+}
+
+func (bm blockMath) mask() int64 {
+	return (1 << bm.blockSizeBits) - 1
+}
+
+// BlockSize returns the block size.
+func (bm blockMath) BlockSize() int {
+	return 1 << bm.blockSizeBits
+}
+
+// Block returns the block index containing the given offset.
+func (bm blockMath) Block(offset int64) cacheBlockIndex {
+	return cacheBlockIndex(offset >> bm.blockSizeBits)
+}
+
+// Remainder returns the offset relative to the start of the cache block.
+func (bm blockMath) Remainder(offset int64) int64 {
+	return offset & bm.mask()
+}
+
+// BlockOffset returns the object offset where the given block starts.
+func (bm blockMath) BlockOffset(block cacheBlockIndex) int64 {
+	return int64(block) << bm.blockSizeBits
+}
+
+// RoundUp rounds up the given value to the closest multiple of block size.
+func (bm blockMath) RoundUp(x int64) int64 {
+	return (x + bm.mask()) & ^(bm.mask())
+}
+
+type writeWorkers struct {
+	doneCh        chan struct{}
+	doneWaitGroup sync.WaitGroup
+
+	numWorkers int
+	tasksCh    chan writeTask
+}
+
+type writeTask struct {
+	fileNum base.DiskFileNum
+	p       []byte
+	offset  int64
+}
+
+// Start starts the worker goroutines.
+func (w *writeWorkers) Start(c *Cache, numWorkers int) {
+	doneCh := make(chan struct{})
+	tasksCh := make(chan writeTask, numWorkers*writeTasksPerWorker)
+
+	w.numWorkers = numWorkers
+	w.doneCh = doneCh
+	w.tasksCh = tasksCh
+	w.doneWaitGroup.Add(numWorkers)
+	for i := 0; i < numWorkers; i++ {
+		go func() {
+			defer w.doneWaitGroup.Done()
+			for {
+				select {
+				case <-doneCh:
+					return
+				case task, ok := <-tasksCh:
+					if !ok {
+						// The tasks channel was closed; this is used in testing code to
+						// ensure all writes are completed.
+						return
+					}
+					// TODO(radu): set() can perform multiple writes; perhaps each one
+					// should be its own task.
+					start := time.Now()
+					err := c.set(task.fileNum, task.p, task.offset)
+					c.metrics.putLatency.Observe(float64(time.Since(start)))
+					if err != nil {
+						c.metrics.writeBackFailures.Add(1)
+						// TODO(radu): throttle logs.
+						c.logger.Infof("writing back to cache after miss failed: %v", err)
+					}
+				}
+			}
+		}()
+	}
+}
+
+// Stop waits for any in-progress writes to complete and stops the worker
+// goroutines and waits for any in-pro. Any queued writes not yet started are
+// discarded.
+func (w *writeWorkers) Stop() {
+	close(w.doneCh)
+	w.doneCh = nil
+	w.tasksCh = nil
+	w.doneWaitGroup.Wait()
+}
+
+// QueueWrite adds a write task to the queue. Can block if the queue is full.
+func (w *writeWorkers) QueueWrite(fileNum base.DiskFileNum, p []byte, offset int64) {
+	w.tasksCh <- writeTask{
+		fileNum: fileNum,
+		p:       p,
+		offset:  offset,
+	}
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs.go

@@ -2,63 +2,63 @@
 // of this source code is governed by a BSD-style license that can be found in
 // the LICENSE file.
 
-package objstorage
+package objstorageprovider
 
 import (
+	"context"
+
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/vfs"
 )
 
-func (p *Provider) vfsPath(fileType base.FileType, fileNum base.FileNum) string {
+func (p *provider) vfsPath(fileType base.FileType, fileNum base.DiskFileNum) string {
 	return base.MakeFilepath(p.st.FS, p.st.FSDirName, fileType, fileNum)
 }
 
-func (p *Provider) vfsOpenForReading(
+func (p *provider) vfsOpenForReading(
-	fileType base.FileType, fileNum base.FileNum, mustExist bool,
+	ctx context.Context,
-) (Readable, error) {
+	fileType base.FileType,
+	fileNum base.DiskFileNum,
+	opts objstorage.OpenOptions,
+) (objstorage.Readable, error) {
 	filename := p.vfsPath(fileType, fileNum)
 	file, err := p.st.FS.Open(filename, vfs.RandomReadsOption)
 	if err != nil {
-		if mustExist {
+		if opts.MustExist {
 			base.MustExist(p.st.FS, filename, p.st.Logger, err)
 		}
 		return nil, err
 	}
-	// TODO(radu): we use the existence of the file descriptor as an indication
+	return newFileReadable(file, p.st.FS, filename)
-	// that the File might support Prefetch and SequentialReadsOption. We should
-	// replace this with a cleaner way to obtain the capabilities of the FS / File.
-	if fd := file.Fd(); fd != vfs.InvalidFd {
-		return newFileReadable(file, p.st.FS, filename)
-	}
-	return newGenericFileReadable(file)
 }
 
-func (p *Provider) vfsCreate(
+func (p *provider) vfsCreate(
-	fileType base.FileType, fileNum base.FileNum,
+	_ context.Context, fileType base.FileType, fileNum base.DiskFileNum,
-) (Writable, ObjectMetadata, error) {
+) (objstorage.Writable, objstorage.ObjectMetadata, error) {
 	filename := p.vfsPath(fileType, fileNum)
 	file, err := p.st.FS.Create(filename)
 	if err != nil {
-		return nil, ObjectMetadata{}, err
+		return nil, objstorage.ObjectMetadata{}, err
 	}
 	file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{
 		NoSyncOnClose: p.st.NoSyncOnClose,
 		BytesPerSync:  p.st.BytesPerSync,
 	})
-	meta := ObjectMetadata{
+	meta := objstorage.ObjectMetadata{
-		FileNum:  fileNum,
+		DiskFileNum: fileNum,
-		FileType: fileType,
+		FileType:    fileType,
 	}
 	return newFileBufferedWritable(file), meta, nil
 }
 
-func (p *Provider) vfsRemove(fileType base.FileType, fileNum base.FileNum) error {
+func (p *provider) vfsRemove(fileType base.FileType, fileNum base.DiskFileNum) error {
 	return p.st.FSCleaner.Clean(p.st.FS, fileType, p.vfsPath(fileType, fileNum))
 }
 
 // vfsInit finds any local FS objects.
-func (p *Provider) vfsInit() error {
+func (p *provider) vfsInit() error {
 	listing := p.st.FSDirInitialListing
 	if listing == nil {
 		var err error
@@ -71,17 +71,17 @@ func (p *Provider) vfsInit() error {
 	for _, filename := range listing {
 		fileType, fileNum, ok := base.ParseFilename(p.st.FS, filename)
 		if ok && fileType == base.FileTypeTable {
-			o := ObjectMetadata{
+			o := objstorage.ObjectMetadata{
-				FileType: fileType,
+				FileType:    fileType,
-				FileNum:  fileNum,
+				DiskFileNum: fileNum,
 			}
-			p.mu.knownObjects[o.FileNum] = o
+			p.mu.knownObjects[o.DiskFileNum] = o
 		}
 	}
 	return nil
 }
 
-func (p *Provider) vfsSync() error {
+func (p *provider) vfsSync() error {
 	p.mu.Lock()
 	shouldSync := p.mu.localObjectsChanged
 	p.mu.localObjectsChanged = false
@@ -99,7 +99,7 @@ func (p *Provider) vfsSync() error {
 	return nil
 }
 
-func (p *Provider) vfsSize(fileType base.FileType, fileNum base.FileNum) (int64, error) {
+func (p *provider) vfsSize(fileType base.FileType, fileNum base.DiskFileNum) (int64, error) {
 	filename := p.vfsPath(fileType, fileNum)
 	stat, err := p.st.FS.Stat(filename)
 	if err != nil {

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go

@@ -2,19 +2,25 @@
 // of this source code is governed by a BSD-style license that can be found in
 // the LICENSE file.
 
-package objstorage
+package objstorageprovider
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"sync"
 
 	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/vfs"
 )
 
-// fileReadable implements objstorage.Readable on top of a vfs.File that
+const fileMaxReadaheadSize = 256 * 1024 /* 256KB */
-// supports Prefetch and/or on an FS that supports SequentialReadsOption.
+
+// fileReadable implements objstorage.Readable on top of a vfs.File.
+//
+// The implementation might use Prealloc and might reopen the file with
+// SequentialReadsOption.
 type fileReadable struct {
 	file vfs.File
 	size int64
@@ -25,7 +31,7 @@ type fileReadable struct {
 	fs       vfs.FS
 }
 
-var _ Readable = (*fileReadable)(nil)
+var _ objstorage.Readable = (*fileReadable)(nil)
 
 func newFileReadable(file vfs.File, fs vfs.FS, filename string) (*fileReadable, error) {
 	info, err := file.Stat()
@@ -48,8 +54,12 @@ func newFileReadable(file vfs.File, fs vfs.FS, filename string) (*fileReadable,
 }
 
 // ReadAt is part of the objstorage.Readable interface.
-func (r *fileReadable) ReadAt(p []byte, off int64) (n int, err error) {
+func (r *fileReadable) ReadAt(_ context.Context, p []byte, off int64) error {
-	return r.file.ReadAt(p, off)
+	n, err := r.file.ReadAt(p, off)
+	if invariants.Enabled && err == nil && n != len(p) {
+		panic("short read")
+	}
+	return err
 }
 
 // Close is part of the objstorage.Readable interface.
@@ -64,10 +74,10 @@ func (r *fileReadable) Size() int64 {
 }
 
 // NewReadHandle is part of the objstorage.Readable interface.
-func (r *fileReadable) NewReadHandle() ReadHandle {
+func (r *fileReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
 	rh := readHandlePool.Get().(*vfsReadHandle)
 	rh.r = r
-	rh.rs = makeReadaheadState()
+	rh.rs = makeReadaheadState(fileMaxReadaheadSize)
 	return rh
 }
 
@@ -82,7 +92,7 @@ type vfsReadHandle struct {
 	sequentialFile vfs.File
 }
 
-var _ ReadHandle = (*vfsReadHandle)(nil)
+var _ objstorage.ReadHandle = (*vfsReadHandle)(nil)
 
 var readHandlePool = sync.Pool{
 	New: func() interface{} {
@@ -110,25 +120,36 @@ func (rh *vfsReadHandle) Close() error {
 }
 
 // ReadAt is part of the objstorage.ReadHandle interface.
-func (rh *vfsReadHandle) ReadAt(p []byte, offset int64) (n int, err error) {
+func (rh *vfsReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error {
+	var n int
+	var err error
 	if rh.sequentialFile != nil {
 		// Use OS-level read-ahead.
-		return rh.sequentialFile.ReadAt(p, offset)
+		n, err = rh.sequentialFile.ReadAt(p, offset)
-	}
+	} else {
-	if readaheadSize := rh.rs.maybeReadahead(offset, int64(len(p))); readaheadSize > 0 {
+		if readaheadSize := rh.rs.maybeReadahead(offset, int64(len(p))); readaheadSize > 0 {
-		if readaheadSize >= maxReadaheadSize {
+			if readaheadSize >= fileMaxReadaheadSize {
-			// We've reached the maximum readahead size. Beyond this point, rely on
+				// We've reached the maximum readahead size. Beyond this point, rely on
-			// OS-level readahead.
+				// OS-level readahead.
-			rh.MaxReadahead()
+				rh.switchToOSReadahead()
-		} else {
+			} else {
-			_ = rh.r.file.Prefetch(offset, readaheadSize)
+				_ = rh.r.file.Prefetch(offset, readaheadSize)
+			}
 		}
+		n, err = rh.r.file.ReadAt(p, offset)
 	}
-	return rh.r.file.ReadAt(p, offset)
+	if invariants.Enabled && err == nil && n != len(p) {
+		panic("short read")
+	}
+	return err
 }
 
-// MaxReadahead is part of the objstorage.ReadHandle interface.
+// SetupForCompaction is part of the objstorage.ReadHandle interface.
-func (rh *vfsReadHandle) MaxReadahead() {
+func (rh *vfsReadHandle) SetupForCompaction() {
+	rh.switchToOSReadahead()
+}
+
+func (rh *vfsReadHandle) switchToOSReadahead() {
 	if rh.sequentialFile != nil {
 		return
 	}
@@ -142,7 +163,7 @@ func (rh *vfsReadHandle) MaxReadahead() {
 }
 
 // RecordCacheHit is part of the objstorage.ReadHandle interface.
-func (rh *vfsReadHandle) RecordCacheHit(offset, size int64) {
+func (rh *vfsReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
 	if rh.sequentialFile != nil {
 		// Using OS-level readahead, so do nothing.
 		return
@@ -150,60 +171,9 @@ func (rh *vfsReadHandle) RecordCacheHit(offset, size int64) {
 	rh.rs.recordCacheHit(offset, size)
 }
 
-// genericFileReadable implements objstorage.Readable on top of any vfs.File.
-// This implementation does not support read-ahead.
-type genericFileReadable struct {
-	file vfs.File
-	size int64
-
-	rh NoopReadHandle
-}
-
-var _ Readable = (*genericFileReadable)(nil)
-
-func newGenericFileReadable(file vfs.File) (*genericFileReadable, error) {
-	info, err := file.Stat()
-	if err != nil {
-		return nil, err
-	}
-	r := &genericFileReadable{
-		file: file,
-		size: info.Size(),
-		rh:   MakeNoopReadHandle(file),
-	}
-	invariants.SetFinalizer(r, func(obj interface{}) {
-		if obj.(*genericFileReadable).file != nil {
-			fmt.Fprintf(os.Stderr, "Readable was not closed")
-			os.Exit(1)
-		}
-	})
-	return r, nil
-}
-
-// ReadAt is part of the objstorage.Readable interface.
-func (r *genericFileReadable) ReadAt(p []byte, off int64) (n int, err error) {
-	return r.file.ReadAt(p, off)
-}
-
-// Close is part of the objstorage.Readable interface.
-func (r *genericFileReadable) Close() error {
-	defer func() { r.file = nil }()
-	return r.file.Close()
-}
-
-// Size is part of the objstorage.Readable interface.
-func (r *genericFileReadable) Size() int64 {
-	return r.size
-}
-
-// NewReadHandle is part of the objstorage.Readable interface.
-func (r *genericFileReadable) NewReadHandle() ReadHandle {
-	return &r.rh
-}
-
 // TestingCheckMaxReadahead returns true if the ReadHandle has switched to
 // OS-level read-ahead.
-func TestingCheckMaxReadahead(rh ReadHandle) bool {
+func TestingCheckMaxReadahead(rh objstorage.ReadHandle) bool {
 	switch rh := rh.(type) {
 	case *vfsReadHandle:
 		return rh.sequentialFile != nil
@@ -234,11 +204,13 @@ func (rh *PreallocatedReadHandle) Close() error {
 // but uses the existing storage of a PreallocatedReadHandle when possible
 // (currently this happens if we are reading from a local file).
 // The returned handle still needs to be closed.
-func UsePreallocatedReadHandle(readable Readable, rh *PreallocatedReadHandle) ReadHandle {
+func UsePreallocatedReadHandle(
+	ctx context.Context, readable objstorage.Readable, rh *PreallocatedReadHandle,
+) objstorage.ReadHandle {
 	if r, ok := readable.(*fileReadable); ok {
 		// See fileReadable.NewReadHandle.
 		rh.vfsReadHandle = vfsReadHandle{r: r}
 		return rh
 	}
-	return readable.NewReadHandle()
+	return readable.NewReadHandle(ctx)
 }

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_writable.go

@@ -2,16 +2,17 @@
 // of this source code is governed by a BSD-style license that can be found in
 // the LICENSE file.
 
-package objstorage
+package objstorageprovider
 
 import (
 	"bufio"
 
+	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/vfs"
 )
 
 // NewFileWritable returns a Writable that uses a file as underlying storage.
-func NewFileWritable(file vfs.File) Writable {
+func NewFileWritable(file vfs.File) objstorage.Writable {
 	return newFileBufferedWritable(file)
 }
 
@@ -20,7 +21,7 @@ type fileBufferedWritable struct {
 	bw   *bufio.Writer
 }
 
-var _ Writable = (*fileBufferedWritable)(nil)
+var _ objstorage.Writable = (*fileBufferedWritable)(nil)
 
 func newFileBufferedWritable(file vfs.File) *fileBufferedWritable {
 	return &fileBufferedWritable{

vendor/github.com/cockroachdb/pebble/objstorage/provider.go

@@ -1,446 +0,0 @@
-// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
-// of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package objstorage
-
-import (
-	"io"
-	"os"
-	"sort"
-	"sync"
-
-	"github.com/cockroachdb/errors"
-	"github.com/cockroachdb/errors/oserror"
-	"github.com/cockroachdb/pebble/internal/base"
-	"github.com/cockroachdb/pebble/objstorage/shared"
-	"github.com/cockroachdb/pebble/objstorage/sharedobjcat"
-	"github.com/cockroachdb/pebble/vfs"
-)
-
-// Provider is a singleton object used to access and manage objects.
-//
-// An object is conceptually like a large immutable file. The main use of
-// objects is for storing sstables; in the future it could also be used for blob
-// storage.
-//
-// The Provider can only manage objects that it knows about - either objects
-// created by the provider, or existing objects the Provider was informed about
-// via AddObjects.
-//
-// Objects are currently backed by a vfs.File.
-type Provider struct {
-	st Settings
-
-	fsDir vfs.File
-
-	shared sharedSubsystem
-
-	mu struct {
-		sync.RWMutex
-
-		shared struct {
-			// catalogBatch accumulates shared object creations and deletions until
-			// Sync is called.
-			catalogBatch sharedobjcat.Batch
-		}
-
-		// localObjectsChanged is set if non-shared objects were created or deleted
-		// but Sync was not yet called.
-		localObjectsChanged bool
-
-		// knownObjects maintains information about objects that are known to the provider.
-		// It is initialized with the list of files in the manifest when we open a DB.
-		knownObjects map[base.FileNum]ObjectMetadata
-	}
-}
-
-// Readable is the handle for an object that is open for reading.
-type Readable interface {
-	io.ReaderAt
-	io.Closer
-
-	// Size returns the size of the object.
-	Size() int64
-
-	// NewReadHandle creates a read handle for ReadAt requests that are related
-	// and can benefit from optimizations like read-ahead.
-	//
-	// The ReadHandle must be closed before the Readable is closed.
-	//
-	// Multiple separate ReadHandles can be used.
-	NewReadHandle() ReadHandle
-}
-
-// ReadHandle is used to perform reads that are related and might benefit from
-// optimizations like read-ahead.
-type ReadHandle interface {
-	io.ReaderAt
-	io.Closer
-
-	// MaxReadahead configures the implementation to expect large sequential
-	// reads. Used to skip any initial read-ahead ramp-up.
-	MaxReadahead()
-
-	// RecordCacheHit informs the implementation that we were able to retrieve a
-	// block from cache.
-	RecordCacheHit(offset, size int64)
-}
-
-// Writable is the handle for an object that is open for writing.
-// Either Finish or Abort must be called.
-type Writable interface {
-	// Write writes len(p) bytes from p to the underlying object. The data is not
-	// guaranteed to be durable until Finish is called.
-	//
-	// Note that Write *is* allowed to modify the slice passed in, whether
-	// temporarily or permanently. Callers of Write need to take this into
-	// account.
-	Write(p []byte) error
-
-	// Finish completes the object and makes the data durable.
-	// No further calls are allowed after calling Finish.
-	Finish() error
-
-	// Abort gives up on finishing the object. There is no guarantee about whether
-	// the object exists after calling Abort.
-	// No further calls are allowed after calling Abort.
-	Abort()
-}
-
-// Settings that must be specified when creating the Provider.
-type Settings struct {
-	Logger base.Logger
-
-	// Local filesystem configuration.
-	FS        vfs.FS
-	FSDirName string
-
-	// FSDirInitialListing is a listing of FSDirName at the time of calling Open.
-	//
-	// This is an optional optimization to avoid double listing on Open when the
-	// higher layer already has a listing. When nil, we obtain the listing on
-	// Open.
-	FSDirInitialListing []string
-
-	// Cleaner cleans obsolete files from the local filesystem.
-	//
-	// The default cleaner uses the DeleteCleaner.
-	FSCleaner base.Cleaner
-
-	// NoSyncOnClose decides whether the implementation will enforce a
-	// close-time synchronization (e.g., fdatasync() or sync_file_range())
-	// on files it writes to. Setting this to true removes the guarantee for a
-	// sync on close. Some implementations can still issue a non-blocking sync.
-	NoSyncOnClose bool
-
-	// BytesPerSync enables periodic syncing of files in order to smooth out
-	// writes to disk. This option does not provide any persistence guarantee, but
-	// is used to avoid latency spikes if the OS automatically decides to write
-	// out a large chunk of dirty filesystem buffers.
-	BytesPerSync int
-
-	// Fields here are set only if the provider is to support shared objects
-	// (experimental).
-	Shared struct {
-		Storage shared.Storage
-	}
-}
-
-// DefaultSettings initializes default settings (with no shared storage),
-// suitable for tests and tools.
-func DefaultSettings(fs vfs.FS, dirName string) Settings {
-	return Settings{
-		Logger:        base.DefaultLogger,
-		FS:            fs,
-		FSDirName:     dirName,
-		FSCleaner:     base.DeleteCleaner{},
-		NoSyncOnClose: false,
-		BytesPerSync:  512 * 1024, // 512KB
-	}
-}
-
-// ObjectMetadata contains the metadata required to be able to access an object.
-type ObjectMetadata struct {
-	FileNum  base.FileNum
-	FileType base.FileType
-
-	// The fields below are only set if the object is on shared storage.
-	Shared struct {
-		// CreatorID identifies the DB instance that originally created the object.
-		CreatorID CreatorID
-		// CreatorFileNum is the identifier for the object within the context of the
-		// DB instance that originally created the object.
-		CreatorFileNum base.FileNum
-	}
-}
-
-// CreatorID identifies the DB instance that originally created a shared object.
-// This ID is incorporated in backing object names.
-// Must be non-zero.
-type CreatorID = sharedobjcat.CreatorID
-
-// IsShared returns true if the object is on shared storage.
-func (meta *ObjectMetadata) IsShared() bool {
-	return meta.Shared.CreatorID.IsSet()
-}
-
-// Open creates the Provider.
-func Open(settings Settings) (p *Provider, _ error) {
-	fsDir, err := settings.FS.OpenDir(settings.FSDirName)
-	if err != nil {
-		return nil, err
-	}
-
-	defer func() {
-		if p == nil {
-			fsDir.Close()
-		}
-	}()
-
-	p = &Provider{
-		st:    settings,
-		fsDir: fsDir,
-	}
-	p.mu.knownObjects = make(map[base.FileNum]ObjectMetadata)
-
-	// Add local FS objects.
-	if err := p.vfsInit(); err != nil {
-		return nil, err
-	}
-
-	// Initialize shared subsystem (if configured) and add shared objects.
-	if err := p.sharedInit(); err != nil {
-		return nil, err
-	}
-
-	return p, nil
-}
-
-// Close the provider.
-func (p *Provider) Close() error {
-	var err error
-	if p.fsDir != nil {
-		err = p.fsDir.Close()
-		p.fsDir = nil
-	}
-	return err
-}
-
-// OpenForReading opens an existing object.
-func (p *Provider) OpenForReading(fileType base.FileType, fileNum base.FileNum) (Readable, error) {
-	meta, err := p.Lookup(fileType, fileNum)
-	if err != nil {
-		return nil, err
-	}
-
-	if !meta.IsShared() {
-		return p.vfsOpenForReading(fileType, fileNum, false /* mustExist */)
-	}
-	return p.sharedOpenForReading(meta)
-}
-
-// OpenForReadingMustExist is a variant of OpenForReading which causes a fatal
-// error if the file does not exist. The fatal error message contains
-// information helpful for debugging.
-func (p *Provider) OpenForReadingMustExist(
-	fileType base.FileType, fileNum base.FileNum,
-) (Readable, error) {
-	meta, err := p.Lookup(fileType, fileNum)
-	if err != nil {
-		p.st.Logger.Fatalf("%v", err)
-		return nil, err
-	}
-
-	if !meta.IsShared() {
-		return p.vfsOpenForReading(fileType, fileNum, true /* mustExist */)
-	}
-
-	// TODO(radu): implement "must exist" behavior.
-	return p.sharedOpenForReading(meta)
-}
-
-// CreateOptions contains optional arguments for Create.
-type CreateOptions struct {
-	// PreferSharedStorage causes the object to be created on shared storage if
-	// the provider has shared storage configured.
-	PreferSharedStorage bool
-}
-
-// Create creates a new object and opens it for writing.
-//
-// The object is not guaranteed to be durable (accessible in case of crashes)
-// until Sync is called.
-func (p *Provider) Create(
-	fileType base.FileType, fileNum base.FileNum, opts CreateOptions,
-) (w Writable, meta ObjectMetadata, err error) {
-	if opts.PreferSharedStorage && p.st.Shared.Storage != nil {
-		w, meta, err = p.sharedCreate(fileType, fileNum)
-	} else {
-		w, meta, err = p.vfsCreate(fileType, fileNum)
-	}
-	if err != nil {
-		err = errors.Wrapf(err, "creating object %s", errors.Safe(fileNum))
-		return nil, ObjectMetadata{}, err
-	}
-	p.addMetadata(meta)
-	return w, meta, nil
-}
-
-// Remove removes an object.
-//
-// The object is not guaranteed to be durably removed until Sync is called.
-func (p *Provider) Remove(fileType base.FileType, fileNum base.FileNum) error {
-	meta, err := p.Lookup(fileType, fileNum)
-	if err != nil {
-		return err
-	}
-
-	if !meta.IsShared() {
-		err = p.vfsRemove(fileType, fileNum)
-	}
-	// TODO(radu): implement shared object removal (i.e. deref).
-
-	if err != nil && !IsNotExistError(err) {
-		// We want to be able to retry a Remove, so we keep the object in our list.
-		// TODO(radu): we should mark the object as "zombie" and not allow any other
-		// operations.
-		return errors.Wrapf(err, "removing object %s", errors.Safe(fileNum))
-	}
-	p.removeMetadata(fileNum)
-	return err
-}
-
-// IsNotExistError indicates whether the error is known to report that a file or
-// directory does not exist.
-func IsNotExistError(err error) bool {
-	return oserror.IsNotExist(err)
-}
-
-// Sync flushes the metadata from creation or removal of objects since the last Sync.
-func (p *Provider) Sync() error {
-	if err := p.vfsSync(); err != nil {
-		return err
-	}
-	if err := p.sharedSync(); err != nil {
-		return err
-	}
-	return nil
-}
-
-// LinkOrCopyFromLocal creates a new object that is either a copy of a given
-// local file or a hard link (if the new object is created on the same FS, and
-// if the FS supports it).
-//
-// The object is not guaranteed to be durable (accessible in case of crashes)
-// until Sync is called.
-func (p *Provider) LinkOrCopyFromLocal(
-	srcFS vfs.FS, srcFilePath string, dstFileType base.FileType, dstFileNum base.FileNum,
-) (ObjectMetadata, error) {
-	if srcFS == p.st.FS {
-		// Wrap the normal filesystem with one which wraps newly created files with
-		// vfs.NewSyncingFile.
-		fs := vfs.NewSyncingFS(p.st.FS, vfs.SyncingFileOptions{
-			NoSyncOnClose: p.st.NoSyncOnClose,
-			BytesPerSync:  p.st.BytesPerSync,
-		})
-		dstPath := p.vfsPath(dstFileType, dstFileNum)
-		if err := vfs.LinkOrCopy(fs, srcFilePath, dstPath); err != nil {
-			return ObjectMetadata{}, err
-		}
-
-		meta := ObjectMetadata{
-			FileNum:  dstFileNum,
-			FileType: dstFileType,
-		}
-		p.addMetadata(meta)
-		return meta, nil
-	}
-	// TODO(radu): for the copy case, we should use `p.Create` and do the copy ourselves.
-	panic("unimplemented")
-}
-
-// Lookup returns the metadata of an object that is already known to the Provider.
-// Does not perform any I/O.
-func (p *Provider) Lookup(fileType base.FileType, fileNum base.FileNum) (ObjectMetadata, error) {
-	p.mu.RLock()
-	defer p.mu.RUnlock()
-	meta, ok := p.mu.knownObjects[fileNum]
-	if !ok {
-		return ObjectMetadata{}, errors.Wrapf(
-			os.ErrNotExist,
-			"file %s (type %d) unknown to the objstorage provider",
-			errors.Safe(fileNum), errors.Safe(fileType),
-		)
-	}
-	if meta.FileType != fileType {
-		return ObjectMetadata{}, errors.AssertionFailedf(
-			"file %s type mismatch (known type %d, expected type %d)",
-			errors.Safe(fileNum), errors.Safe(meta.FileType), errors.Safe(fileType),
-		)
-	}
-	return meta, nil
-}
-
-// Path returns an internal, implementation-dependent path for the object. It is
-// meant to be used for informational purposes (like logging).
-func (p *Provider) Path(meta ObjectMetadata) string {
-	if !meta.IsShared() {
-		return p.vfsPath(meta.FileType, meta.FileNum)
-	}
-	return p.sharedPath(meta)
-}
-
-// Size returns the size of the object.
-func (p *Provider) Size(meta ObjectMetadata) (int64, error) {
-	if !meta.IsShared() {
-		return p.vfsSize(meta.FileType, meta.FileNum)
-	}
-	return p.sharedSize(meta)
-}
-
-// List returns the objects currently known to the provider. Does not perform any I/O.
-func (p *Provider) List() []ObjectMetadata {
-	p.mu.RLock()
-	defer p.mu.RUnlock()
-	res := make([]ObjectMetadata, 0, len(p.mu.knownObjects))
-	for _, meta := range p.mu.knownObjects {
-		res = append(res, meta)
-	}
-	sort.Slice(res, func(i, j int) bool {
-		return res[i].FileNum < res[j].FileNum
-	})
-	return res
-}
-
-func (p *Provider) addMetadata(meta ObjectMetadata) {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-	p.mu.knownObjects[meta.FileNum] = meta
-	if meta.IsShared() {
-		p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
-			FileNum:        meta.FileNum,
-			FileType:       meta.FileType,
-			CreatorID:      meta.Shared.CreatorID,
-			CreatorFileNum: meta.Shared.CreatorFileNum,
-		})
-	} else {
-		p.mu.localObjectsChanged = true
-	}
-}
-
-func (p *Provider) removeMetadata(fileNum base.FileNum) {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	meta, ok := p.mu.knownObjects[fileNum]
-	if !ok {
-		return
-	}
-	delete(p.mu.knownObjects, fileNum)
-	if meta.IsShared() {
-		p.mu.shared.catalogBatch.DeleteObject(fileNum)
-	} else {
-		p.mu.localObjectsChanged = true
-	}
-}

vendor/github.com/cockroachdb/pebble/objstorage/remote/factory.go

@@ -0,0 +1,25 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package remote
+
+import "github.com/pkg/errors"
+
+// MakeSimpleFactory returns a StorageFactory implementation that produces the given
+// Storage objects.
+func MakeSimpleFactory(m map[Locator]Storage) StorageFactory {
+	return simpleFactory(m)
+}
+
+type simpleFactory map[Locator]Storage
+
+var _ StorageFactory = simpleFactory{}
+
+// CreateStorage is part of the StorageFactory interface.
+func (sf simpleFactory) CreateStorage(locator Locator) (Storage, error) {
+	if s, ok := sf[locator]; ok {
+		return s, nil
+	}
+	return nil, errors.Errorf("unknown locator '%s'", locator)
+}

vendor/github.com/cockroachdb/pebble/objstorage/remote/localfs.go

@@ -0,0 +1,118 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package remote
+
+import (
+	"context"
+	"io"
+	"os"
+	"path"
+
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// NewLocalFS returns a vfs-backed implementation of the remote.Storage
+// interface (for testing). All objects will be stored at the directory
+// dirname.
+func NewLocalFS(dirname string, fs vfs.FS) Storage {
+	store := &localFSStore{
+		dirname: dirname,
+		vfs:     fs,
+	}
+	return store
+}
+
+// localFSStore is a vfs-backed implementation of the remote.Storage
+// interface (for testing).
+type localFSStore struct {
+	dirname string
+	vfs     vfs.FS
+}
+
+var _ Storage = (*localFSStore)(nil)
+
+// Close is part of the remote.Storage interface.
+func (s *localFSStore) Close() error {
+	*s = localFSStore{}
+	return nil
+}
+
+// ReadObject is part of the remote.Storage interface.
+func (s *localFSStore) ReadObject(
+	ctx context.Context, objName string,
+) (_ ObjectReader, objSize int64, _ error) {
+	f, err := s.vfs.Open(path.Join(s.dirname, objName))
+	if err != nil {
+		return nil, 0, err
+	}
+	stat, err := f.Stat()
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return &localFSReader{f}, stat.Size(), nil
+}
+
+type localFSReader struct {
+	file vfs.File
+}
+
+var _ ObjectReader = (*localFSReader)(nil)
+
+// ReadAt is part of the shared.ObjectReader interface.
+func (r *localFSReader) ReadAt(_ context.Context, p []byte, offset int64) error {
+	n, err := r.file.ReadAt(p, offset)
+	// https://pkg.go.dev/io#ReaderAt
+	if err == io.EOF && n == len(p) {
+		return nil
+	}
+	return err
+}
+
+// Close is part of the shared.ObjectReader interface.
+func (r *localFSReader) Close() error {
+	r.file.Close()
+	r.file = nil
+	return nil
+}
+
+// CreateObject is part of the remote.Storage interface.
+func (s *localFSStore) CreateObject(objName string) (io.WriteCloser, error) {
+	file, err := s.vfs.Create(path.Join(s.dirname, objName))
+	return file, err
+}
+
+// List is part of the remote.Storage interface.
+func (s *localFSStore) List(prefix, delimiter string) ([]string, error) {
+	// TODO(josh): For the intended use case of localfs.go of running 'pebble bench',
+	// List can always return <nil, nil>, since this indicates a file has only one ref,
+	// and since `pebble bench` implies running in a single-pebble-instance context.
+	// https://github.com/cockroachdb/pebble/blob/a9a079d4fb6bf4a9ebc52e4d83a76ad4cbf676cb/objstorage/objstorageprovider/shared.go#L292
+	return nil, nil
+}
+
+// Delete is part of the remote.Storage interface.
+func (s *localFSStore) Delete(objName string) error {
+	return s.vfs.Remove(path.Join(s.dirname, objName))
+}
+
+// Size is part of the remote.Storage interface.
+func (s *localFSStore) Size(objName string) (int64, error) {
+	f, err := s.vfs.Open(path.Join(s.dirname, objName))
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	stat, err := f.Stat()
+	if err != nil {
+		return 0, err
+	}
+	return stat.Size(), nil
+}
+
+// IsNotExistError is part of the remote.Storage interface.
+func (s *localFSStore) IsNotExistError(err error) bool {
+	return err == os.ErrNotExist
+}

vendor/github.com/cockroachdb/pebble/objstorage/remote/logging.go

@@ -2,9 +2,10 @@
 // of this source code is governed by a BSD-style license that can be found in
 // the LICENSE file.
 
-package shared
+package remote
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"sort"
@@ -19,7 +20,7 @@ func WithLogging(wrapped Storage, logf func(fmt string, args ...interface{})) St
 	}
 }
 
-// loggingStore wraps a shared.Storage implementation and emits logs of the
+// loggingStore wraps a remote.Storage implementation and emits logs of the
 // operations.
 type loggingStore struct {
 	logf    func(fmt string, args ...interface{})
@@ -33,41 +34,52 @@ func (l *loggingStore) Close() error {
 	return l.wrapped.Close()
 }
 
-func (l *loggingStore) ReadObjectAt(
+func (l *loggingStore) ReadObject(
-	basename string, offset int64,
+	ctx context.Context, objName string,
-) (_ io.ReadCloser, totalSize int64, _ error) {
+) (_ ObjectReader, objSize int64, _ error) {
-	r, totalSize, err := l.wrapped.ReadObjectAt(basename, offset)
+	r, size, err := l.wrapped.ReadObject(ctx, objName)
-	l.logf("read object %q at %d: %s", basename, offset, errOrPrintf(err, "%d bytes", totalSize))
+	l.logf("create reader for object %q: %s", objName, errOrPrintf(err, "%d bytes", size))
 	if err != nil {
 		return nil, 0, err
 	}
 	return &loggingReader{
-		l:          l,
+		l:       l,
-		name:       basename,
+		name:    objName,
-		ReadCloser: r,
+		wrapped: r,
-	}, totalSize, nil
+	}, size, nil
 }
 
 type loggingReader struct {
-	l    *loggingStore
+	l       *loggingStore
-	name string
+	name    string
-	io.ReadCloser
+	wrapped ObjectReader
+}
+
+var _ ObjectReader = (*loggingReader)(nil)
+
+func (l *loggingReader) ReadAt(ctx context.Context, p []byte, offset int64) error {
+	if err := l.wrapped.ReadAt(ctx, p, offset); err != nil {
+		l.l.logf("read object %q at %d (length %d): error %v", l.name, offset, len(p), err)
+		return err
+	}
+	l.l.logf("read object %q at %d (length %d)", l.name, offset, len(p))
+	return nil
 }
 
 func (l *loggingReader) Close() error {
 	l.l.logf("close reader for %q", l.name)
-	return l.ReadCloser.Close()
+	return l.wrapped.Close()
 }
 
-func (l *loggingStore) CreateObject(basename string) (io.WriteCloser, error) {
+func (l *loggingStore) CreateObject(objName string) (io.WriteCloser, error) {
-	l.logf("create object %q", basename)
+	l.logf("create object %q", objName)
-	writer, err := l.wrapped.CreateObject(basename)
+	writer, err := l.wrapped.CreateObject(objName)
 	if err != nil {
 		return nil, err
 	}
 	return &loggingWriter{
 		l:           l,
-		name:        basename,
+		name:        objName,
 		WriteCloser: writer,
 	}, nil
 }
@@ -104,14 +116,14 @@ func (l *loggingStore) List(prefix, delimiter string) ([]string, error) {
 	return res, nil
 }
 
-func (l *loggingStore) Delete(basename string) error {
+func (l *loggingStore) Delete(objName string) error {
-	l.logf("delete object %q", basename)
+	l.logf("delete object %q", objName)
-	return l.wrapped.Delete(basename)
+	return l.wrapped.Delete(objName)
 }
 
-func (l *loggingStore) Size(basename string) (int64, error) {
+func (l *loggingStore) Size(objName string) (int64, error) {
-	size, err := l.wrapped.Size(basename)
+	size, err := l.wrapped.Size(objName)
-	l.logf("size of object %q: %s", basename, errOrPrintf(err, "%d", size))
+	l.logf("size of object %q: %s", objName, errOrPrintf(err, "%d", size))
 	return size, err
 }
 
@@ -121,3 +133,7 @@ func errOrPrintf(err error, format string, args ...interface{}) string {
 	}
 	return fmt.Sprintf(format, args...)
 }
+
+func (l *loggingStore) IsNotExistError(err error) bool {
+	return l.wrapped.IsNotExistError(err)
+}

vendor/github.com/cockroachdb/pebble/objstorage/remote/mem.go

@@ -2,17 +2,18 @@
 // of this source code is governed by a BSD-style license that can be found in
 // the LICENSE file.
 
-package shared
+package remote
 
 import (
 	"bytes"
+	"context"
 	"io"
 	"os"
 	"strings"
 	"sync"
 )
 
-// NewInMem returns an in-memory implementation of the shared.Storage
+// NewInMem returns an in-memory implementation of the remote.Storage
 // interface (for testing).
 func NewInMem() Storage {
 	store := &inMemStore{}
@@ -20,7 +21,7 @@ func NewInMem() Storage {
 	return store
 }
 
-// inMemStore is an in-memory implementation of the shared.Storage interface
+// inMemStore is an in-memory implementation of the remote.Storage interface
 // (for testing).
 type inMemStore struct {
 	mu struct {
@@ -41,39 +42,39 @@ func (s *inMemStore) Close() error {
 	return nil
 }
 
-func (s *inMemStore) ReadObjectAt(basename string, offset int64) (io.ReadCloser, int64, error) {
+func (s *inMemStore) ReadObject(
-	obj, err := s.getObj(basename)
+	ctx context.Context, objName string,
+) (_ ObjectReader, objSize int64, _ error) {
+	obj, err := s.getObj(objName)
 	if err != nil {
 		return nil, 0, err
 	}
-	remaining := int64(len(obj.data)) - offset
+	return &inMemReader{data: obj.data}, int64(len(obj.data)), nil
-	if remaining < 0 {
-		return nil, 0, io.EOF
-	}
-	return newInMemReader(obj.data[int(offset):]), remaining, nil
 }
 
 type inMemReader struct {
-	bytes.Reader
+	data []byte
 }
 
-func newInMemReader(b []byte) *inMemReader {
+var _ ObjectReader = (*inMemReader)(nil)
-	r := &inMemReader{}
-	r.Reset(b)
-	return r
-}
 
-var _ io.ReadCloser = (*inMemReader)(nil)
+func (r *inMemReader) ReadAt(ctx context.Context, p []byte, offset int64) error {
-
+	if offset+int64(len(p)) > int64(len(r.data)) {
-func (r *inMemReader) Close() error {
+		return io.EOF
-	r.Reset(nil)
+	}
+	copy(p, r.data[offset:])
 	return nil
 }
 
-func (s *inMemStore) CreateObject(basename string) (io.WriteCloser, error) {
+func (r *inMemReader) Close() error {
+	r.data = nil
+	return nil
+}
+
+func (s *inMemStore) CreateObject(objName string) (io.WriteCloser, error) {
 	return &inMemWriter{
 		store: s,
-		name:  basename,
+		name:  objName,
 	}, nil
 }
 
@@ -119,20 +120,24 @@ func (s *inMemStore) List(prefix, delimiter string) ([]string, error) {
 	return res, nil
 }
 
-func (s *inMemStore) Delete(basename string) error {
+func (s *inMemStore) Delete(objName string) error {
-	s.rmObj(basename)
+	s.rmObj(objName)
 	return nil
 }
 
 // Size returns the length of the named object in bytesWritten.
-func (s *inMemStore) Size(basename string) (int64, error) {
+func (s *inMemStore) Size(objName string) (int64, error) {
-	obj, err := s.getObj(basename)
+	obj, err := s.getObj(objName)
 	if err != nil {
 		return 0, err
 	}
 	return int64(len(obj.data)), nil
 }
 
+func (s *inMemStore) IsNotExistError(err error) bool {
+	return err == os.ErrNotExist
+}
+
 func (s *inMemStore) getObj(name string) (*inMemObj, error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()

vendor/github.com/cockroachdb/pebble/objstorage/remote/storage.go

@@ -0,0 +1,126 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package remote
+
+import (
+	"context"
+	"io"
+)
+
+// Locator is an opaque string identifying a remote.Storage implementation.
+//
+// The Locator must not contain secrets (like authentication keys). Locators are
+// stored on disk in the shared object catalog and are passed around as part of
+// RemoteObjectBacking; they can also appear in error messages.
+type Locator string
+
+// StorageFactory is used to return Storage implementations based on locators. A
+// Pebble store that uses shared storage is configured with a StorageFactory.
+type StorageFactory interface {
+	CreateStorage(locator Locator) (Storage, error)
+}
+
+// SharedLevelsStart denotes the highest (i.e. lowest numbered) level that will
+// have sstables shared across Pebble instances when doing skip-shared
+// iteration (see db.ScanInternal) or shared file ingestion (see
+// db.IngestAndExcise).
+const SharedLevelsStart = 5
+
+// CreateOnSharedStrategy specifies what table files should be created on shared
+// storage. For use with CreateOnShared in options.
+type CreateOnSharedStrategy int
+
+const (
+	// CreateOnSharedNone denotes no files being created on shared storage.
+	CreateOnSharedNone CreateOnSharedStrategy = iota
+	// CreateOnSharedLower denotes the creation of files in lower levels of the
+	// LSM (specifically, L5 and L6 as they're below SharedLevelsStart) on
+	// shared storage, and higher levels on local storage.
+	CreateOnSharedLower
+	// CreateOnSharedAll denotes the creation of all sstables on shared storage.
+	CreateOnSharedAll
+)
+
+// ShouldCreateShared returns whether new table files at the specified level
+// should be created on shared storage.
+func ShouldCreateShared(strategy CreateOnSharedStrategy, level int) bool {
+	switch strategy {
+	case CreateOnSharedAll:
+		return true
+	case CreateOnSharedNone:
+		return false
+	case CreateOnSharedLower:
+		return level >= SharedLevelsStart
+	default:
+		panic("unexpected CreateOnSharedStrategy value")
+	}
+}
+
+// Storage is an interface for a blob storage driver. This is lower-level
+// than an FS-like interface, however FS/File-like abstractions can be built on
+// top of these methods.
+//
+// TODO(bilal): Consider pushing shared file obsoletion as well as path
+// generation behind this interface.
+type Storage interface {
+	io.Closer
+
+	// ReadObject returns an ObjectReader that can be used to perform reads on an
+	// object, along with the total size of the object.
+	ReadObject(ctx context.Context, objName string) (_ ObjectReader, objSize int64, _ error)
+
+	// CreateObject returns a writer for the object at the request name. A new
+	// empty object is created if CreateObject is called on an existing object.
+	//
+	// A Writer *must* be closed via either Close, and if closing returns a
+	// non-nil error, that error should be handled or reported to the user -- an
+	// implementation may buffer written data until Close and only then return
+	// an error, or Write may return an opaque io.EOF with the underlying cause
+	// returned by the subsequent Close().
+	//
+	// TODO(radu): if we encounter some unrelated error while writing to the
+	// WriteCloser, we'd want to abort the whole thing rather than letting Close
+	// finalize the upload.
+	CreateObject(objName string) (io.WriteCloser, error)
+
+	// List enumerates files within the supplied prefix, returning a list of
+	// objects within that prefix. If delimiter is non-empty, names which have the
+	// same prefix, prior to the delimiter but after the prefix, are grouped into a
+	// single result which is that prefix. The order that results are returned is
+	// undefined. If a prefix is specified, the prefix is trimmed from the result
+	// list.
+	//
+	// An example would be, if the storage contains objects a, b/4, b/5 and b/6,
+	// these would be the return values:
+	//   List("", "") -> ["a", "b/4", "b/5", "b/6"]
+	//   List("", "/") -> ["a", "b"]
+	//   List("b", "/") -> ["4", "5", "6"]
+	//   List("b", "") -> ["/4", "/5", "/6"]
+	List(prefix, delimiter string) ([]string, error)
+
+	// Delete removes the named object from the store.
+	Delete(objName string) error
+
+	// Size returns the length of the named object in bytesWritten.
+	Size(objName string) (int64, error)
+
+	// IsNotExistError returns true if the given error (returned by a method in
+	// this interface) indicates that the object does not exist.
+	IsNotExistError(err error) bool
+}
+
+// ObjectReader is used to perform reads on an object.
+type ObjectReader interface {
+	// ReadAt reads len(p) bytes into p starting at offset off.
+	//
+	// Does not return partial results; if offset + len(p) is past the end of the
+	// object, an error is returned.
+	//
+	// Clients of ReadAt can execute parallel ReadAt calls on the same
+	// ObjectReader.
+	ReadAt(ctx context.Context, p []byte, offset int64) error
+
+	Close() error
+}

vendor/github.com/cockroachdb/pebble/objstorage/shared.go

@@ -1,176 +0,0 @@
-// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
-// of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package objstorage
-
-import (
-	"fmt"
-	"sync"
-	"sync/atomic"
-
-	"github.com/cockroachdb/errors"
-	"github.com/cockroachdb/pebble/internal/base"
-	"github.com/cockroachdb/pebble/objstorage/sharedobjcat"
-)
-
-// sharedSubsystem contains the provider fields related to shared storage.
-// All fields remain unset if shared storage is not configured.
-type sharedSubsystem struct {
-	catalog *sharedobjcat.Catalog
-	// initialized guards access to the creatorID field.
-	initialized atomic.Bool
-	creatorID   CreatorID
-	initOnce    sync.Once
-}
-
-func (ss *sharedSubsystem) init(creatorID CreatorID) {
-	ss.initOnce.Do(func() {
-		ss.creatorID = creatorID
-		ss.initialized.Store(true)
-	})
-}
-
-// sharedInit initializes the shared object subsystem (if configured) and finds
-// any shared objects.
-func (p *Provider) sharedInit() error {
-	if p.st.Shared.Storage == nil {
-		return nil
-	}
-	catalog, contents, err := sharedobjcat.Open(p.st.FS, p.st.FSDirName)
-	if err != nil {
-		return errors.Wrapf(err, "pebble: could not open shared object catalog")
-	}
-	p.shared.catalog = catalog
-
-	// The creator ID may or may not be initialized yet.
-	if contents.CreatorID.IsSet() {
-		p.shared.init(contents.CreatorID)
-		p.st.Logger.Infof("shared storage configured; creatorID = %s", contents.CreatorID)
-	} else {
-		p.st.Logger.Infof("shared storage configured; no creatorID yet")
-	}
-
-	for _, meta := range contents.Objects {
-		o := ObjectMetadata{
-			FileNum:  meta.FileNum,
-			FileType: meta.FileType,
-		}
-		o.Shared.CreatorID = meta.CreatorID
-		o.Shared.CreatorFileNum = meta.CreatorFileNum
-		p.mu.knownObjects[o.FileNum] = o
-	}
-	return nil
-}
-
-// SetCreatorID sets the CreatorID which is needed in order to use shared
-// objects. Shared object usage is disabled until this method is called the
-// first time. Once set, the Creator ID is persisted and cannot change.
-//
-// Cannot be called if shared storage is not configured for the provider.
-func (p *Provider) SetCreatorID(creatorID CreatorID) error {
-	if p.st.Shared.Storage == nil {
-		return errors.AssertionFailedf("attempt to set CreatorID but shared storage not enabled")
-	}
-	// Note: this call is a cheap no-op if the creator ID was already set. This
-	// call also checks if we are trying to change the ID.
-	if err := p.shared.catalog.SetCreatorID(creatorID); err != nil {
-		return err
-	}
-	if !p.shared.initialized.Load() {
-		p.st.Logger.Infof("shared storage creatorID set to %s", creatorID)
-		p.shared.init(creatorID)
-	}
-	return nil
-}
-
-func (p *Provider) sharedCheckInitialized() error {
-	if p.st.Shared.Storage == nil {
-		return errors.Errorf("shared object support not configured")
-	}
-	if !p.shared.initialized.Load() {
-		return errors.Errorf("shared object support not available: shared creator ID not yet set")
-	}
-	return nil
-}
-
-func (p *Provider) sharedSync() error {
-	batch := func() sharedobjcat.Batch {
-		p.mu.Lock()
-		defer p.mu.Unlock()
-		res := p.mu.shared.catalogBatch.Copy()
-		p.mu.shared.catalogBatch.Reset()
-		return res
-	}()
-
-	if batch.IsEmpty() {
-		return nil
-	}
-
-	err := p.shared.catalog.ApplyBatch(batch)
-	if err != nil {
-		// We have to put back the batch (for the next Sync).
-		p.mu.Lock()
-		defer p.mu.Unlock()
-		batch.Append(p.mu.shared.catalogBatch)
-		p.mu.shared.catalogBatch = batch
-		return err
-	}
-
-	return nil
-}
-
-func (p *Provider) sharedPath(meta ObjectMetadata) string {
-	return "shared://" + sharedObjectName(meta)
-}
-
-func sharedObjectName(meta ObjectMetadata) string {
-	// TODO(radu): prepend a "shard" value for better distribution within the bucket?
-	return fmt.Sprintf(
-		"%s-%s",
-		meta.Shared.CreatorID, base.MakeFilename(meta.FileType, meta.Shared.CreatorFileNum),
-	)
-}
-
-func (p *Provider) sharedCreate(
-	fileType base.FileType, fileNum base.FileNum,
-) (Writable, ObjectMetadata, error) {
-	if err := p.sharedCheckInitialized(); err != nil {
-		return nil, ObjectMetadata{}, err
-	}
-	meta := ObjectMetadata{
-		FileNum:  fileNum,
-		FileType: fileType,
-	}
-	meta.Shared.CreatorID = p.shared.creatorID
-	meta.Shared.CreatorFileNum = fileNum
-
-	objName := sharedObjectName(meta)
-	writer, err := p.st.Shared.Storage.CreateObject(objName)
-	if err != nil {
-		return nil, ObjectMetadata{}, err
-	}
-	return &sharedWritable{
-		storageWriter: writer,
-	}, meta, nil
-}
-
-func (p *Provider) sharedOpenForReading(meta ObjectMetadata) (Readable, error) {
-	if err := p.sharedCheckInitialized(); err != nil {
-		return nil, err
-	}
-	objName := sharedObjectName(meta)
-	size, err := p.st.Shared.Storage.Size(objName)
-	if err != nil {
-		return nil, err
-	}
-	return newSharedReadable(p.st.Shared.Storage, objName, size), nil
-}
-
-func (p *Provider) sharedSize(meta ObjectMetadata) (int64, error) {
-	if err := p.sharedCheckInitialized(); err != nil {
-		return 0, err
-	}
-	objName := sharedObjectName(meta)
-	return p.st.Shared.Storage.Size(objName)
-}

vendor/github.com/cockroachdb/pebble/objstorage/shared/storage.go

@@ -1,61 +0,0 @@
-// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
-// of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package shared
-
-import "io"
-
-// Storage is an interface for a blob storage driver. This is lower-level
-// than an FS-like interface, however FS/File-like abstractions can be built on
-// top of these methods.
-//
-// TODO(bilal): Consider pushing shared file obsoletion as well as path
-// generation behind this interface.
-type Storage interface {
-	io.Closer
-
-	// ReadObjectAt returns a Reader for reading the object at the requested name
-	// and offset, along with the total size of the object.
-	ReadObjectAt(basename string, offset int64) (_ io.ReadCloser, totalSize int64, _ error)
-
-	// CreateObject returns a writer for the object at the request name. A new
-	// empty object is created if CreateObject is called on an existing object.
-	//
-	// A Writer *must* be closed via either Close, and if closing returns a
-	// non-nil error, that error should be handled or reported to the user -- an
-	// implementation may buffer written data until Close and only then return
-	// an error, or Write may return an opaque io.EOF with the underlying cause
-	// returned by the subsequent Close().
-	//
-	// TODO(radu): if we encounter some unrelated error while writing to the
-	// WriteCloser, we'd want to abort the whole thing rather than letting Close
-	// finalize the upload.
-	CreateObject(basename string) (io.WriteCloser, error)
-
-	// List enumerates files within the supplied prefix, returning a list of
-	// objects within that prefix. If delimiter is non-empty, names which have the
-	// same prefix, prior to the delimiter but after the prefix, are grouped into a
-	// single result which is that prefix. The order that results are returned is
-	// undefined. If a prefix is specified, the prefix is trimmed from the result
-	// list.
-	//
-	// An example would be, if the storage contains objects a, b/4, b/5 and b/6,
-	// these would be the return values:
-	//   List("", "") -> ["a", "b/4", "b/5", "b/6"]
-	//   List("", "/") -> ["a", "b"]
-	//   List("b", "/") -> ["4", "5", "6"]
-	//   List("b", "") -> ["/4", "/5", "/6"]
-	List(prefix, delimiter string) ([]string, error)
-
-	// Delete removes the named object from the store.
-	//
-	// TODO(radu): do we get an error when the object isn't there? How can we
-	// check for that error?
-	Delete(basename string) error
-
-	// Size returns the length of the named object in bytesWritten.
-	//
-	// TODO(radu): same as above - how can we tell if it's a "no such object" error?
-	Size(basename string) (int64, error)
-}

vendor/github.com/cockroachdb/pebble/objstorage/shared_backing.go

@@ -1,145 +0,0 @@
-// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
-// of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package objstorage
-
-import (
-	"bytes"
-	"encoding/binary"
-	"io"
-
-	"github.com/cockroachdb/errors"
-	"github.com/cockroachdb/pebble/internal/base"
-	"github.com/cockroachdb/pebble/objstorage/sharedobjcat"
-)
-
-// SharedObjectBacking encodes the metadata necessary to incorporate a shared
-// object into a different Pebble instance.
-type SharedObjectBacking []byte
-
-const (
-	tagCreatorID      = 1
-	tagCreatorFileNum = 2
-
-	// Any new tags that don't have the tagNotSafeToIgnoreMask bit set must be
-	// followed by the length of the data (so they can be skipped).
-
-	// Any new tags that have the tagNotSafeToIgnoreMask bit set cause errors if
-	// they are encountered by earlier code that doesn't know the tag.
-	tagNotSafeToIgnoreMask = 64
-)
-
-// SharedObjectBacking encodes the shared object metadata.
-func (meta *ObjectMetadata) SharedObjectBacking() (SharedObjectBacking, error) {
-	if !meta.IsShared() {
-		return nil, errors.AssertionFailedf("object %s not on shared storage", meta.FileNum)
-	}
-
-	buf := make([]byte, 0, binary.MaxVarintLen64*4)
-	buf = binary.AppendUvarint(buf, tagCreatorID)
-	buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorID))
-	buf = binary.AppendUvarint(buf, tagCreatorFileNum)
-	buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorFileNum))
-	return buf, nil
-}
-
-// fromSharedObjectBacking decodes the shared object metadata.
-func fromSharedObjectBacking(
-	fileType base.FileType, fileNum base.FileNum, buf SharedObjectBacking,
-) (ObjectMetadata, error) {
-	var creatorID uint64
-	var creatorFileNum uint64
-	br := bytes.NewReader(buf)
-	for {
-		tag, err := binary.ReadUvarint(br)
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return ObjectMetadata{}, err
-		}
-		switch tag {
-		case tagCreatorID:
-			creatorID, err = binary.ReadUvarint(br)
-
-		case tagCreatorFileNum:
-			creatorFileNum, err = binary.ReadUvarint(br)
-
-		// TODO(radu): encode file type as well?
-
-		default:
-			// Ignore unknown tags, unless they're not safe to ignore.
-			if tag&tagNotSafeToIgnoreMask != 0 {
-				return ObjectMetadata{}, errors.Newf("unknown tag %d", tag)
-			}
-			var dataLen uint64
-			dataLen, err = binary.ReadUvarint(br)
-			if err == nil {
-				_, err = br.Seek(int64(dataLen), io.SeekCurrent)
-			}
-		}
-		if err != nil {
-			return ObjectMetadata{}, err
-		}
-	}
-	if creatorID == 0 {
-		return ObjectMetadata{}, errors.Newf("shared object backing missing creator ID")
-	}
-	if creatorFileNum == 0 {
-		return ObjectMetadata{}, errors.Newf("shared object backing missing creator file num")
-	}
-	meta := ObjectMetadata{
-		FileNum:  fileNum,
-		FileType: fileType,
-	}
-	meta.Shared.CreatorID = CreatorID(creatorID)
-	meta.Shared.CreatorFileNum = base.FileNum(creatorFileNum)
-	return meta, nil
-}
-
-// SharedObjectToAttach contains the arguments needed to attach an existing shared object.
-type SharedObjectToAttach struct {
-	// FileNum is the file number that will be used to refer to this object (in
-	// the context of this instance).
-	FileNum  base.FileNum
-	FileType base.FileType
-	// Backing contains the metadata for the share dobject backing (normally
-	// generated from a different instance).
-	Backing SharedObjectBacking
-}
-
-// AttachSharedObjects registers existing shared objects with this provider.
-func (p *Provider) AttachSharedObjects(objs []SharedObjectToAttach) ([]ObjectMetadata, error) {
-	metas := make([]ObjectMetadata, len(objs))
-	for i, o := range objs {
-		meta, err := fromSharedObjectBacking(o.FileType, o.FileNum, o.Backing)
-		if err != nil {
-			return nil, err
-		}
-		metas[i] = meta
-	}
-
-	func() {
-		p.mu.Lock()
-		defer p.mu.Unlock()
-		for _, meta := range metas {
-			p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
-				FileNum:        meta.FileNum,
-				FileType:       meta.FileType,
-				CreatorID:      meta.Shared.CreatorID,
-				CreatorFileNum: meta.Shared.CreatorFileNum,
-			})
-		}
-	}()
-	if err := p.sharedSync(); err != nil {
-		return nil, err
-	}
-
-	p.mu.Lock()
-	defer p.mu.Unlock()
-	for _, meta := range metas {
-		p.mu.knownObjects[meta.FileNum] = meta
-	}
-	return metas, nil
-}

vendor/github.com/cockroachdb/pebble/objstorage/shared_readable.go

@@ -1,97 +0,0 @@
-// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
-// of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package objstorage
-
-import (
-	"io"
-
-	"github.com/cockroachdb/pebble/objstorage/shared"
-)
-
-// sharedReadable is a very simple implementation of Readable on top of the
-// ReadCloser returned by shared.Storage.CreateObject.
-type sharedReadable struct {
-	storage shared.Storage
-	objName string
-	size    int64
-
-	// rh is used for direct ReadAt calls without a read handle.
-	rh sharedReadHandle
-}
-
-var _ Readable = (*sharedReadable)(nil)
-
-func newSharedReadable(storage shared.Storage, objName string, size int64) *sharedReadable {
-	r := &sharedReadable{
-		storage: storage,
-		objName: objName,
-		size:    size,
-	}
-	r.rh.readable = r
-	return r
-}
-
-func (r *sharedReadable) ReadAt(p []byte, offset int64) (n int, err error) {
-	return r.rh.ReadAt(p, offset)
-}
-
-func (r *sharedReadable) Close() error {
-	err := r.rh.Close()
-	r.storage = nil
-	return err
-}
-
-func (r *sharedReadable) Size() int64 {
-	return r.size
-}
-
-func (r *sharedReadable) NewReadHandle() ReadHandle {
-	// TODO(radu): use a pool.
-	return &sharedReadHandle{readable: r}
-}
-
-type sharedReadHandle struct {
-	readable   *sharedReadable
-	lastReader io.ReadCloser
-	lastOffset int64
-}
-
-var _ ReadHandle = (*sharedReadHandle)(nil)
-
-func (r *sharedReadHandle) ReadAt(p []byte, offset int64) (n int, err error) {
-	// See if this continues the previous read so that we can reuse the last reader.
-	if r.lastReader == nil || r.lastOffset != offset {
-		// We need to create a new reader.
-		if r.lastReader != nil {
-			if err := r.lastReader.Close(); err != nil {
-				return 0, err
-			}
-			r.lastReader = nil
-		}
-		reader, _, err := r.readable.storage.ReadObjectAt(r.readable.objName, offset)
-		if err != nil {
-			return 0, err
-		}
-		r.lastReader = reader
-		r.lastOffset = offset
-	}
-	n, err = io.ReadFull(r.lastReader, p)
-	r.lastOffset += int64(n)
-	return n, err
-}
-
-func (r *sharedReadHandle) Close() error {
-	var err error
-	if r.lastReader != nil {
-		err = r.lastReader.Close()
-		r.lastReader = nil
-	}
-	r.readable = nil
-	return err
-}
-
-func (r *sharedReadHandle) MaxReadahead() {}
-
-func (r *sharedReadHandle) RecordCacheHit(offset, size int64) {}

vendor/github.com/cockroachdb/pebble/objstorage/shared_writable.go

@@ -1,34 +0,0 @@
-// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
-// of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package objstorage
-
-import "io"
-
-// sharedWritable is a very simple implementation of Writable on top of the
-// WriteCloser returned by shared.Storage.CreateObject.
-type sharedWritable struct {
-	storageWriter io.WriteCloser
-}
-
-var _ Writable = (*sharedWritable)(nil)
-
-// Write is part of the Writable interface.
-func (w *sharedWritable) Write(p []byte) error {
-	_, err := w.storageWriter.Write(p)
-	return err
-}
-
-// Finish is part of the Writable interface.
-func (w *sharedWritable) Finish() error {
-	err := w.storageWriter.Close()
-	w.storageWriter = nil
-	return err
-}
-
-// Abort is part of the Writable interface.
-func (w *sharedWritable) Abort() {
-	_ = w.storageWriter.Close()
-	w.storageWriter = nil
-}

vendor/github.com/cockroachdb/pebble/objstorage/sharedobjcat/version_edit.go

@@ -1,158 +0,0 @@
-// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
-// of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package sharedobjcat
-
-import (
-	"bufio"
-	"encoding/binary"
-	"io"
-
-	"github.com/cockroachdb/errors"
-	"github.com/cockroachdb/pebble/internal/base"
-)
-
-// versionEdit is a modification to the shared object state which can be encoded
-// into a record.
-//
-// TODO(radu): consider adding creation and deletion time for debugging purposes.
-type versionEdit struct {
-	NewObjects     []SharedObjectMetadata
-	DeletedObjects []base.FileNum
-	CreatorID      CreatorID
-}
-
-const (
-	// tagNewObject is followed by the FileNum, creator ID and creator FileNum.
-	tagNewObject = 1
-	// tagDeletedObject is followed by the FileNum.
-	tagDeletedObject = 2
-	// tagCreatorID is followed by the Creator ID for this store. This ID can
-	// never change.
-	tagCreatorID = 3
-)
-
-// Object type values. We don't want to encode FileType directly because it is
-// more general (and we want freedom to change it in the future).
-const (
-	objTypeTable = 1
-)
-
-func objTypeToFileType(objType uint64) (base.FileType, error) {
-	switch objType {
-	case objTypeTable:
-		return base.FileTypeTable, nil
-	default:
-		return 0, errors.Newf("unknown object type %d", objType)
-	}
-}
-
-func fileTypeToObjType(fileType base.FileType) (uint64, error) {
-	switch fileType {
-	case base.FileTypeTable:
-		return objTypeTable, nil
-
-	default:
-		return 0, errors.Newf("unknown object type for file type %d", fileType)
-	}
-}
-
-// Encode encodes an edit to the specified writer.
-func (v *versionEdit) Encode(w io.Writer) error {
-	buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*4+len(v.DeletedObjects)*2+2))
-	for _, meta := range v.NewObjects {
-		objType, err := fileTypeToObjType(meta.FileType)
-		if err != nil {
-			return err
-		}
-		buf = binary.AppendUvarint(buf, uint64(tagNewObject))
-		buf = binary.AppendUvarint(buf, uint64(meta.FileNum))
-		buf = binary.AppendUvarint(buf, objType)
-		buf = binary.AppendUvarint(buf, uint64(meta.CreatorID))
-		buf = binary.AppendUvarint(buf, uint64(meta.CreatorFileNum))
-	}
-
-	for _, fileNum := range v.DeletedObjects {
-		buf = binary.AppendUvarint(buf, uint64(tagDeletedObject))
-		buf = binary.AppendUvarint(buf, uint64(fileNum))
-	}
-	if v.CreatorID.IsSet() {
-		buf = binary.AppendUvarint(buf, uint64(tagCreatorID))
-		buf = binary.AppendUvarint(buf, uint64(v.CreatorID))
-	}
-	_, err := w.Write(buf)
-	return err
-}
-
-// Decode decodes an edit from the specified reader.
-func (v *versionEdit) Decode(r io.Reader) error {
-	br, ok := r.(io.ByteReader)
-	if !ok {
-		br = bufio.NewReader(r)
-	}
-	for {
-		tag, err := binary.ReadUvarint(br)
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return err
-		}
-
-		err = nil
-		switch tag {
-		case tagNewObject:
-			var fileNum, creatorID, creatorFileNum uint64
-			var fileType base.FileType
-			fileNum, err = binary.ReadUvarint(br)
-			if err == nil {
-				var objType uint64
-				objType, err = binary.ReadUvarint(br)
-				if err == nil {
-					fileType, err = objTypeToFileType(objType)
-				}
-			}
-			if err == nil {
-				creatorID, err = binary.ReadUvarint(br)
-			}
-			if err == nil {
-				creatorFileNum, err = binary.ReadUvarint(br)
-			}
-			if err == nil {
-				v.NewObjects = append(v.NewObjects, SharedObjectMetadata{
-					FileNum:        base.FileNum(fileNum),
-					FileType:       fileType,
-					CreatorID:      CreatorID(creatorID),
-					CreatorFileNum: base.FileNum(creatorFileNum),
-				})
-			}
-
-		case tagDeletedObject:
-			var fileNum uint64
-			fileNum, err = binary.ReadUvarint(br)
-			if err == nil {
-				v.DeletedObjects = append(v.DeletedObjects, base.FileNum(fileNum))
-			}
-
-		case tagCreatorID:
-			var id uint64
-			id, err = binary.ReadUvarint(br)
-			if err == nil {
-				v.CreatorID = CreatorID(id)
-			}
-
-		default:
-			// Ignore unknown tags.
-		}
-		if err != nil {
-			if err == io.EOF {
-				return errCorruptCatalog
-			}
-			return err
-		}
-	}
-	return nil
-}
-
-var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt shared object catalog")

vendor/github.com/cockroachdb/pebble/open.go

@@ -6,6 +6,7 @@ package pebble
 
 import (
 	"bytes"
+	"context"
 	"encoding/binary"
 	"fmt"
 	"io"
@@ -20,11 +21,14 @@ import (
 	"github.com/cockroachdb/pebble/internal/arenaskl"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/cache"
+	"github.com/cockroachdb/pebble/internal/constants"
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/cockroachdb/pebble/internal/manual"
 	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
 	"github.com/cockroachdb/pebble/record"
+	"github.com/cockroachdb/pebble/sstable"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -34,11 +38,23 @@ const (
 
 	// The max batch size is limited by the uint32 offsets stored in
 	// internal/batchskl.node, DeferredBatchOp, and flushableBatchEntry.
-	maxBatchSize = 4 << 30 // 4 GB
+	//
+	// We limit the size to MaxUint32 (just short of 4GB) so that the exclusive
+	// end of an allocation fits in uint32.
+	//
+	// On 32-bit systems, slices are naturally limited to MaxInt (just short of
+	// 2GB).
+	maxBatchSize = constants.MaxUint32OrInt
 
 	// The max memtable size is limited by the uint32 offsets stored in
 	// internal/arenaskl.node, DeferredBatchOp, and flushableBatchEntry.
-	maxMemTableSize = 4 << 30 // 4 GB
+	//
+	// We limit the size to MaxUint32 (just short of 4GB) so that the exclusive
+	// end of an allocation fits in uint32.
+	//
+	// On 32-bit systems, slices are naturally limited to MaxInt (just short of
+	// 2GB).
+	maxMemTableSize = constants.MaxUint32OrInt
 )
 
 // TableCacheSize can be used to determine the table
@@ -157,7 +173,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 		merge:               opts.Merger.Merge,
 		split:               opts.Comparer.Split,
 		abbreviatedKey:      opts.Comparer.AbbreviatedKey,
-		largeBatchThreshold: (opts.MemTableSize - int(memTableEmptySize)) / 2,
+		largeBatchThreshold: (opts.MemTableSize - uint64(memTableEmptySize)) / 2,
 		fileLock:            fileLock,
 		dataDir:             dataDir,
 		walDir:              walDir,
@@ -166,8 +182,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 		closedCh:            make(chan struct{}),
 	}
 	d.mu.versions = &versionSet{}
-	d.atomic.diskAvailBytes = math.MaxUint64
+	d.diskAvailBytes.Store(math.MaxUint64)
-	d.mu.versions.diskAvailBytes = d.getDiskAvailableBytesCached
 
 	defer func() {
 		// If an error or panic occurs during open, attempt to release the manually
@@ -192,6 +207,9 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 					t.arenaBuf = nil
 				}
 			}
+			if d.cleanupManager != nil {
+				d.cleanupManager.Close()
+			}
 			if d.objProvider != nil {
 				d.objProvider.Close()
 			}
@@ -202,34 +220,30 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 	}()
 
 	d.commit = newCommitPipeline(commitEnv{
-		logSeqNum:     &d.mu.versions.atomic.logSeqNum,
+		logSeqNum:     &d.mu.versions.logSeqNum,
-		visibleSeqNum: &d.mu.versions.atomic.visibleSeqNum,
+		visibleSeqNum: &d.mu.versions.visibleSeqNum,
 		apply:         d.commitApply,
 		write:         d.commitWrite,
 	})
-	if d.opts.Experimental.MinDeletionRate > 0 {
-		d.deletionPacer = newDeletionPacer(int64(d.opts.Experimental.MinDeletionRate), d.getDeletionPacerInfo)
-	} else {
-		d.deletionPacer = nilPacer
-	}
 	d.mu.nextJobID = 1
 	d.mu.mem.nextSize = opts.MemTableSize
 	if d.mu.mem.nextSize > initialMemTableSize {
 		d.mu.mem.nextSize = initialMemTableSize
 	}
-	d.mu.mem.cond.L = &d.mu.Mutex
-	d.mu.cleaner.cond.L = &d.mu.Mutex
 	d.mu.compact.cond.L = &d.mu.Mutex
 	d.mu.compact.inProgress = make(map[*compaction]struct{})
 	d.mu.compact.noOngoingFlushStartTime = time.Now()
 	d.mu.snapshots.init()
-	// logSeqNum is the next sequence number that will be assigned. Start
+	// logSeqNum is the next sequence number that will be assigned.
-	// assigning sequence numbers from 1 to match rocksdb.
+	// Start assigning sequence numbers from base.SeqNumStart to leave
-	d.mu.versions.atomic.logSeqNum = 1
+	// room for reserved sequence numbers (see comments around
+	// SeqNumStart).
+	d.mu.versions.logSeqNum.Store(base.SeqNumStart)
 	d.mu.formatVers.vers.Store(uint64(formatVersion))
 	d.mu.formatVers.marker = formatVersionMarker
 
 	d.timeNow = time.Now
+	d.openedAt = d.timeNow()
 
 	d.mu.Lock()
 	defer d.mu.Unlock()
@@ -246,7 +260,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 		}
 
 		// Create the DB.
-		if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, setCurrent, &d.mu.Mutex); err != nil {
+		if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, setCurrent, d.FormatMajorVersion, &d.mu.Mutex); err != nil {
 			return nil, err
 		}
 	} else {
@@ -254,11 +268,11 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 			return nil, errors.Wrapf(ErrDBAlreadyExists, "dirname=%q", dirname)
 		}
 		// Load the version set.
-		if err := d.mu.versions.load(dirname, opts, manifestFileNum, manifestMarker, setCurrent, &d.mu.Mutex); err != nil {
+		if err := d.mu.versions.load(dirname, opts, manifestFileNum.FileNum(), manifestMarker, setCurrent, d.FormatMajorVersion, &d.mu.Mutex); err != nil {
 			return nil, err
 		}
 		if opts.ErrorIfNotPristine {
-			liveFileNums := make(map[FileNum]struct{})
+			liveFileNums := make(map[base.DiskFileNum]struct{})
 			d.mu.versions.addLiveFileNums(liveFileNums)
 			if len(liveFileNums) != 0 {
 				return nil, errors.Wrapf(ErrDBNotPristine, "dirname=%q", dirname)
@@ -271,7 +285,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 	// sequence number of the first batch that will be inserted.
 	if !d.opts.ReadOnly {
 		var entry *flushableEntry
-		d.mu.mem.mutable, entry = d.newMemTable(0 /* logNum */, d.mu.versions.atomic.logSeqNum)
+		d.mu.mem.mutable, entry = d.newMemTable(0 /* logNum */, d.mu.versions.logSeqNum.Load())
 		d.mu.mem.queue = append(d.mu.mem.queue, entry)
 	}
 
@@ -287,7 +301,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 		}
 		ls = append(ls, ls2...)
 	}
-	providerSettings := objstorage.Settings{
+	providerSettings := objstorageprovider.Settings{
 		Logger:              opts.Logger,
 		FS:                  opts.FS,
 		FSDirName:           dirname,
@@ -296,13 +310,18 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 		NoSyncOnClose:       opts.NoSyncOnClose,
 		BytesPerSync:        opts.BytesPerSync,
 	}
-	providerSettings.Shared.Storage = opts.Experimental.SharedStorage
+	providerSettings.Remote.StorageFactory = opts.Experimental.RemoteStorage
+	providerSettings.Remote.CreateOnShared = opts.Experimental.CreateOnShared
+	providerSettings.Remote.CreateOnSharedLocator = opts.Experimental.CreateOnSharedLocator
+	providerSettings.Remote.CacheSizeBytes = opts.Experimental.SecondaryCacheSizeBytes
 
-	d.objProvider, err = objstorage.Open(providerSettings)
+	d.objProvider, err = objstorageprovider.Open(providerSettings)
 	if err != nil {
 		return nil, err
 	}
 
+	d.cleanupManager = openCleanupManager(opts, d.objProvider, d.onObsoleteTableDelete, d.getDeletionPacerInfo)
+
 	if manifestExists {
 		curVersion := d.mu.versions.currentVersion()
 		if err := checkConsistency(curVersion, dirname, d.objProvider); err != nil {
@@ -331,21 +350,21 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 
 		// Don't reuse any obsolete file numbers to avoid modifying an
 		// ingested sstable's original external file.
-		if d.mu.versions.nextFileNum <= fn {
+		if d.mu.versions.nextFileNum <= fn.FileNum() {
-			d.mu.versions.nextFileNum = fn + 1
+			d.mu.versions.nextFileNum = fn.FileNum() + 1
 		}
 
 		switch ft {
 		case fileTypeLog:
-			if fn >= d.mu.versions.minUnflushedLogNum {
+			if fn.FileNum() >= d.mu.versions.minUnflushedLogNum {
-				logFiles = append(logFiles, fileNumAndName{fn, filename})
+				logFiles = append(logFiles, fileNumAndName{fn.FileNum(), filename})
 			}
-			if d.logRecycler.minRecycleLogNum <= fn {
+			if d.logRecycler.minRecycleLogNum <= fn.FileNum() {
-				d.logRecycler.minRecycleLogNum = fn + 1
+				d.logRecycler.minRecycleLogNum = fn.FileNum() + 1
 			}
 		case fileTypeOptions:
-			if previousOptionsFileNum < fn {
+			if previousOptionsFileNum < fn.FileNum() {
-				previousOptionsFileNum = fn
+				previousOptionsFileNum = fn.FileNum()
 				previousOptionsFilename = filename
 			}
 		case fileTypeTemp, fileTypeOldTemp:
@@ -362,6 +381,15 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 		}
 	}
 
+	// Ratchet d.mu.versions.nextFileNum ahead of all known objects in the
+	// objProvider. This avoids FileNum collisions with obsolete sstables.
+	objects := d.objProvider.List()
+	for _, obj := range objects {
+		if d.mu.versions.nextFileNum <= obj.DiskFileNum.FileNum() {
+			d.mu.versions.nextFileNum = obj.DiskFileNum.FileNum() + 1
+		}
+	}
+
 	// Validate the most-recent OPTIONS file, if there is one.
 	var strictWALTail bool
 	if previousOptionsFilename != "" {
@@ -387,11 +415,11 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 		}
 		toFlush = append(toFlush, flush...)
 		d.mu.versions.markFileNumUsed(lf.num)
-		if d.mu.versions.atomic.logSeqNum < maxSeqNum {
+		if d.mu.versions.logSeqNum.Load() < maxSeqNum {
-			d.mu.versions.atomic.logSeqNum = maxSeqNum
+			d.mu.versions.logSeqNum.Store(maxSeqNum)
 		}
 	}
-	d.mu.versions.atomic.visibleSeqNum = d.mu.versions.atomic.logSeqNum
+	d.mu.versions.visibleSeqNum.Store(d.mu.versions.logSeqNum.Load())
 
 	if !d.opts.ReadOnly {
 		// Create an empty .log file.
@@ -417,8 +445,8 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 			entry.readerUnrefLocked(true)
 		}
 
-		newLogName := base.MakeFilepath(opts.FS, d.walDirname, fileTypeLog, newLogNum)
+		newLogName := base.MakeFilepath(opts.FS, d.walDirname, fileTypeLog, newLogNum.DiskFileNum())
-		d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum, fileSize: 0})
+		d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum.DiskFileNum(), fileSize: 0})
 		logFile, err := opts.FS.Create(newLogName)
 		if err != nil {
 			return nil, err
@@ -469,7 +497,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 
 	if !d.opts.ReadOnly {
 		// Write the current options to disk.
-		d.optionsFileNum = d.mu.versions.getNextFileNum()
+		d.optionsFileNum = d.mu.versions.getNextFileNum().DiskFileNum()
 		tmpPath := base.MakeFilepath(opts.FS, dirname, fileTypeTemp, d.optionsFileNum)
 		optionsPath := base.MakeFilepath(opts.FS, dirname, fileTypeOptions, d.optionsFileNum)
 
@@ -504,7 +532,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 
 	if !d.opts.ReadOnly {
 		d.scanObsoleteFiles(ls)
-		d.deleteObsoleteFiles(jobID, true /* waitForOngoing */)
+		d.deleteObsoleteFiles(jobID)
 	} else {
 		// All the log files are obsolete.
 		d.mu.versions.metrics.WAL.Files = int64(len(logFiles))
@@ -613,9 +641,9 @@ func GetVersion(dir string, fs vfs.FS) (string, error) {
 			// processed, reset version. This is because rocksdb often
 			// writes multiple options files without deleting previous ones.
 			// Otherwise, skip parsing this options file.
-			if fn > lastOptionsSeen {
+			if fn.FileNum() > lastOptionsSeen {
 				version = ""
-				lastOptionsSeen = fn
+				lastOptionsSeen = fn.FileNum()
 			} else {
 				continue
 			}
@@ -677,8 +705,15 @@ func (d *DB) replayWAL(
 		rr              = record.NewReader(file, logNum)
 		offset          int64 // byte offset in rr
 		lastFlushOffset int64
+		keysReplayed    int64 // number of keys replayed
+		batchesReplayed int64 // number of batches replayed
 	)
 
+	// TODO(jackson): This function is interspersed with panics, in addition to
+	// corruption error propagation. Audit them to ensure we're truly only
+	// panicking where the error points to Pebble bug and not user or
+	// hardware-induced corruption.
+
 	if d.opts.ReadOnly {
 		// In read-only mode, we replay directly into the mutable memtable which will
 		// never be flushed.
@@ -725,7 +760,7 @@ func (d *DB) replayWAL(
 		// TODO(bananabrick): See if we can use the actual base level here,
 		// instead of using 1.
 		c := newFlush(d.opts, d.mu.versions.currentVersion(),
-			1 /* base level */, toFlush)
+			1 /* base level */, toFlush, d.timeNow())
 		newVE, _, _, err := d.runCompaction(jobID, c)
 		if err != nil {
 			return errors.Wrapf(err, "running compaction during WAL replay")
@@ -733,6 +768,11 @@ func (d *DB) replayWAL(
 		ve.NewFiles = append(ve.NewFiles, newVE.NewFiles...)
 		return nil
 	}
+	defer func() {
+		if err != nil {
+			err = errors.WithDetailf(err, "replaying log %s, offset %d", logNum, offset)
+		}
+	}()
 
 	for {
 		offset = rr.Offset()
@@ -765,26 +805,33 @@ func (d *DB) replayWAL(
 
 		// Specify Batch.db so that Batch.SetRepr will compute Batch.memTableSize
 		// which is used below.
-		b = Batch{db: d}
+		b = Batch{}
+		b.db = d
 		b.SetRepr(buf.Bytes())
 		seqNum := b.SeqNum()
 		maxSeqNum = seqNum + uint64(b.Count())
-
+		keysReplayed += int64(b.Count())
+		batchesReplayed++
 		{
 			br := b.Reader()
-			if kind, encodedFileNum, _, _ := br.Next(); kind == InternalKeyKindIngestSST {
+			if kind, encodedFileNum, _, ok, err := br.Next(); err != nil {
-				fileNums := make([]FileNum, 0, b.Count())
+				return nil, 0, err
+			} else if ok && kind == InternalKeyKindIngestSST {
+				fileNums := make([]base.DiskFileNum, 0, b.Count())
 				addFileNum := func(encodedFileNum []byte) {
 					fileNum, n := binary.Uvarint(encodedFileNum)
 					if n <= 0 {
 						panic("pebble: ingest sstable file num is invalid.")
 					}
-					fileNums = append(fileNums, base.FileNum(fileNum))
+					fileNums = append(fileNums, base.FileNum(fileNum).DiskFileNum())
 				}
 				addFileNum(encodedFileNum)
 
 				for i := 1; i < int(b.Count()); i++ {
-					kind, encodedFileNum, _, ok := br.Next()
+					kind, encodedFileNum, _, ok, err := br.Next()
+					if err != nil {
+						return nil, 0, err
+					}
 					if kind != InternalKeyKindIngestSST {
 						panic("pebble: invalid batch key kind.")
 					}
@@ -794,21 +841,41 @@ func (d *DB) replayWAL(
 					addFileNum(encodedFileNum)
 				}
 
-				if _, _, _, ok := br.Next(); ok {
+				if _, _, _, ok, err := br.Next(); err != nil {
+					return nil, 0, err
+				} else if ok {
 					panic("pebble: invalid number of entries in batch.")
 				}
 
-				paths := make([]string, len(fileNums))
+				meta := make([]*fileMetadata, len(fileNums))
 				for i, n := range fileNums {
-					paths[i] = base.MakeFilepath(d.opts.FS, d.dirname, fileTypeTable, n)
+					var readable objstorage.Readable
-				}
+					objMeta, err := d.objProvider.Lookup(fileTypeTable, n)
+					if err != nil {
+						return nil, 0, errors.Wrap(err, "pebble: error when looking up ingested SSTs")
+					}
+					if objMeta.IsRemote() {
+						readable, err = d.objProvider.OpenForReading(context.TODO(), fileTypeTable, n, objstorage.OpenOptions{MustExist: true})
+						if err != nil {
+							return nil, 0, errors.Wrap(err, "pebble: error when opening flushable ingest files")
+						}
+					} else {
+						path := base.MakeFilepath(d.opts.FS, d.dirname, fileTypeTable, n)
+						f, err := d.opts.FS.Open(path)
+						if err != nil {
+							return nil, 0, err
+						}
 
-				var meta []*manifest.FileMetadata
+						readable, err = sstable.NewSimpleReadable(f)
-				meta, _, err = ingestLoad(
+						if err != nil {
-					d.opts, d.FormatMajorVersion(), paths, d.cacheID, fileNums,
+							return nil, 0, err
-				)
+						}
-				if err != nil {
+					}
-					return nil, 0, err
+					// NB: ingestLoad1 will close readable.
+					meta[i], err = ingestLoad1(d.opts, d.FormatMajorVersion(), readable, d.cacheID, n)
+					if err != nil {
+						return nil, 0, errors.Wrap(err, "pebble: error when loading flushable ingest files")
+					}
 				}
 
 				if uint32(len(meta)) != b.Count() {
@@ -854,9 +921,10 @@ func (d *DB) replayWAL(
 						d.opts, d.mu.versions.currentVersion(),
 						1, /* base level */
 						[]*flushableEntry{entry},
+						d.timeNow(),
 					)
 					for _, file := range c.flushing[0].flushable.(*ingestedFlushable).files {
-						ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: 0, Meta: file})
+						ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: 0, Meta: file.FileMetadata})
 					}
 				}
 				return toFlush, maxSeqNum, nil
@@ -868,11 +936,14 @@ func (d *DB) replayWAL(
 			// Make a copy of the data slice since it is currently owned by buf and will
 			// be reused in the next iteration.
 			b.data = append([]byte(nil), b.data...)
-			b.flushable = newFlushableBatch(&b, d.opts.Comparer)
+			b.flushable, err = newFlushableBatch(&b, d.opts.Comparer)
+			if err != nil {
+				return nil, 0, err
+			}
 			entry := d.newFlushableEntry(b.flushable, logNum, b.SeqNum())
 			// Disable memory accounting by adding a reader ref that will never be
 			// removed.
-			entry.readerRefs++
+			entry.readerRefs.Add(1)
 			if d.opts.ReadOnly {
 				d.mu.mem.queue = append(d.mu.mem.queue, entry)
 				// We added the flushable batch to the flushable to the queue.
@@ -910,7 +981,10 @@ func (d *DB) replayWAL(
 		}
 		buf.Reset()
 	}
+
+	d.opts.Logger.Infof("[JOB %d] WAL file %s with log number %s stopped reading at offset: %d; replayed %d keys in %d batches", jobID, filename, logNum.String(), offset, keysReplayed, batchesReplayed)
 	flushMem()
+
 	// mem is nil here.
 	if !d.opts.ReadOnly {
 		err = updateVE()
@@ -990,7 +1064,7 @@ func Peek(dirname string, fs vfs.FS) (*DBDesc, error) {
 // LockDirectory may be used to expand the critical section protected by the
 // database lock to include setup before the call to Open.
 func LockDirectory(dirname string, fs vfs.FS) (*Lock, error) {
-	fileLock, err := fs.Lock(base.MakeFilepath(fs, dirname, fileTypeLock, 0))
+	fileLock, err := fs.Lock(base.MakeFilepath(fs, dirname, fileTypeLock, base.FileNum(0).DiskFileNum()))
 	if err != nil {
 		return nil, err
 	}
@@ -1068,28 +1142,42 @@ func IsCorruptionError(err error) bool {
 	return errors.Is(err, base.ErrCorruption)
 }
 
-func checkConsistency(v *manifest.Version, dirname string, objProvider *objstorage.Provider) error {
+func checkConsistency(v *manifest.Version, dirname string, objProvider objstorage.Provider) error {
 	var buf bytes.Buffer
 	var args []interface{}
 
+	dedup := make(map[base.DiskFileNum]struct{})
 	for level, files := range v.Levels {
 		iter := files.Iter()
 		for f := iter.First(); f != nil; f = iter.Next() {
-			meta, err := objProvider.Lookup(base.FileTypeTable, f.FileNum)
+			backingState := f.FileBacking
+			if _, ok := dedup[backingState.DiskFileNum]; ok {
+				continue
+			}
+			dedup[backingState.DiskFileNum] = struct{}{}
+			fileNum := backingState.DiskFileNum
+			fileSize := backingState.Size
+			// We allow foreign objects to have a mismatch between sizes. This is
+			// because we might skew the backing size stored by our objprovider
+			// to prevent us from over-prioritizing this file for compaction.
+			meta, err := objProvider.Lookup(base.FileTypeTable, fileNum)
 			var size int64
 			if err == nil {
+				if objProvider.IsSharedForeign(meta) {
+					continue
+				}
 				size, err = objProvider.Size(meta)
 			}
 			if err != nil {
 				buf.WriteString("L%d: %s: %v\n")
-				args = append(args, errors.Safe(level), errors.Safe(f.FileNum), err)
+				args = append(args, errors.Safe(level), errors.Safe(fileNum), err)
 				continue
 			}
 
-			if size != int64(f.Size) {
+			if size != int64(fileSize) {
 				buf.WriteString("L%d: %s: object size mismatch (%s): %d (disk) != %d (MANIFEST)\n")
-				args = append(args, errors.Safe(level), errors.Safe(f.FileNum), objProvider.Path(meta),
+				args = append(args, errors.Safe(level), errors.Safe(fileNum), objProvider.Path(meta),
-					errors.Safe(size), errors.Safe(f.Size))
+					errors.Safe(size), errors.Safe(fileSize))
 				continue
 			}
 		}

vendor/github.com/cockroachdb/pebble/options.go

@@ -17,8 +17,10 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/cockroachdb/pebble/internal/humanize"
+	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/manifest"
-	"github.com/cockroachdb/pebble/objstorage/shared"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+	"github.com/cockroachdb/pebble/rangekey"
 	"github.com/cockroachdb/pebble/sstable"
 	"github.com/cockroachdb/pebble/vfs"
 )
@@ -117,11 +119,24 @@ type IterOptions struct {
 	// false to skip scanning. This function must be thread-safe since the same
 	// function can be used by multiple iterators, if the iterator is cloned.
 	TableFilter func(userProps map[string]string) bool
+	// SkipPoint may be used to skip over point keys that don't match an
+	// arbitrary predicate during iteration. If set, the Iterator invokes
+	// SkipPoint for keys encountered. If SkipPoint returns true, the iterator
+	// will skip the key without yielding it to the iterator operation in
+	// progress.
+	//
+	// SkipPoint must be a pure function and always return the same result when
+	// provided the same arguments. The iterator may call SkipPoint multiple
+	// times for the same user key.
+	SkipPoint func(userKey []byte) bool
 	// PointKeyFilters can be used to avoid scanning tables and blocks in tables
-	// when iterating over point keys. It is requires that this slice is sorted in
+	// when iterating over point keys. This slice represents an intersection
-	// increasing order of the BlockPropertyFilter.ShortID. This slice represents
+	// across all filters, i.e., all filters must indicate that the block is
-	// an intersection across all filters, i.e., all filters must indicate that the
+	// relevant.
-	// block is relevant.
+	//
+	// Performance note: When len(PointKeyFilters) > 0, the caller should ensure
+	// that cap(PointKeyFilters) is at least len(PointKeyFilters)+1. This helps
+	// avoid allocations in Pebble internal code that mutates the slice.
 	PointKeyFilters []BlockPropertyFilter
 	// RangeKeyFilters can be usefd to avoid scanning tables and blocks in tables
 	// when iterating over range keys. The same requirements that apply to
@@ -180,6 +195,10 @@ type IterOptions struct {
 	level manifest.Level
 	// disableLazyCombinedIteration is an internal testing option.
 	disableLazyCombinedIteration bool
+	// snapshotForHideObsoletePoints is specified for/by levelIter when opening
+	// files and is used to decide whether to hide obsolete points. A value of 0
+	// implies obsolete points should not be hidden.
+	snapshotForHideObsoletePoints uint64
 
 	// NB: If adding new Options, you must account for them in iterator
 	// construction and Iterator.SetOptions.
@@ -222,6 +241,38 @@ func (o *IterOptions) getLogger() Logger {
 	return o.logger
 }
 
+// SpanIterOptions creates a SpanIterOptions from this IterOptions.
+func (o *IterOptions) SpanIterOptions() keyspan.SpanIterOptions {
+	if o == nil {
+		return keyspan.SpanIterOptions{}
+	}
+	return keyspan.SpanIterOptions{
+		RangeKeyFilters: o.RangeKeyFilters,
+	}
+}
+
+// scanInternalOptions is similar to IterOptions, meant for use with
+// scanInternalIterator.
+type scanInternalOptions struct {
+	IterOptions
+
+	visitPointKey   func(key *InternalKey, value LazyValue, iterInfo IteratorLevel) error
+	visitRangeDel   func(start, end []byte, seqNum uint64) error
+	visitRangeKey   func(start, end []byte, keys []rangekey.Key) error
+	visitSharedFile func(sst *SharedSSTMeta) error
+
+	// skipSharedLevels skips levels that are shareable (level >=
+	// sharedLevelStart).
+	skipSharedLevels bool
+
+	// includeObsoleteKeys specifies whether keys shadowed by newer internal keys
+	// are exposed. If false, only one internal key per user key is exposed.
+	includeObsoleteKeys bool
+
+	// rateLimitFunc is used to limit the amount of bytes read per second.
+	rateLimitFunc func(key *InternalKey, value LazyValue) error
+}
+
 // RangeKeyMasking configures automatic hiding of point keys by range keys. A
 // non-nil Suffix enables range-key masking. When enabled, range keys with
 // suffixes ≥ Suffix behave as masks. All point keys that are contained within a
@@ -497,18 +548,12 @@ type Options struct {
 		// concurrent compaction is added. This works "on top" of
 		// L0CompactionConcurrency, so the higher of the count of compaction
 		// concurrency slots as determined by the two options is chosen.
-		CompactionDebtConcurrency int
+		CompactionDebtConcurrency uint64
 
-		// MinDeletionRate is the minimum number of bytes per second that would
+		// IngestSplit, if it returns true, allows for ingest-time splitting of
-		// be deleted. Deletion pacing is used to slow down deletions when
+		// existing sstables into two virtual sstables to allow ingestion sstables to
-		// compactions finish up or readers close, and newly-obsolete files need
+		// slot into a lower level than they otherwise would have.
-		// cleaning up. Deleting lots of files at once can cause disk latency to
+		IngestSplit func() bool
-		// go up on some SSDs, which this functionality guards against. This is a
-		// minimum as the maximum is theoretically unlimited; pacing is disabled
-		// when there are too many obsolete files relative to live bytes, or
-		// there isn't enough disk space available. Setting this to 0 disables
-		// deletion pacing, which is also the default.
-		MinDeletionRate int
 
 		// ReadCompactionRate controls the frequency of read triggered
 		// compactions by adjusting `AllowedSeeks` in manifest.FileMetadata:
@@ -573,10 +618,10 @@ type Options struct {
 		// desired size of each level of the LSM. Defaults to 10.
 		LevelMultiplier int
 
-		// MultiLevelCompactionHueristic determines whether to add an additional
+		// MultiLevelCompactionHeuristic determines whether to add an additional
 		// level to a conventional two level compaction. If nil, a multilevel
 		// compaction will never get triggered.
-		MultiLevelCompactionHueristic MultiLevelHeuristic
+		MultiLevelCompactionHeuristic MultiLevelHeuristic
 
 		// MaxWriterConcurrency is used to indicate the maximum number of
 		// compression workers the compression queue is allowed to use. If
@@ -597,8 +642,11 @@ type Options struct {
 		CPUWorkPermissionGranter CPUWorkPermissionGranter
 
 		// EnableValueBlocks is used to decide whether to enable writing
-		// TableFormatPebblev3 sstables. WARNING: do not return true yet, since
+		// TableFormatPebblev3 sstables. This setting is only respected by a
-		// support for TableFormatPebblev3 is incomplete and not production ready.
+		// specific subset of format major versions: FormatSSTableValueBlocks,
+		// FormatFlushableIngest and FormatPrePebblev1MarkedCompacted. In lower
+		// format major versions, value blocks are never enabled. In higher
+		// format major versions, value blocks are always enabled.
 		EnableValueBlocks func() bool
 
 		// ShortAttributeExtractor is used iff EnableValueBlocks() returns true
@@ -607,18 +655,24 @@ type Options struct {
 		ShortAttributeExtractor ShortAttributeExtractor
 
 		// RequiredInPlaceValueBound specifies an optional span of user key
-		// prefixes for which the values must be stored with the key. This is
+		// prefixes that are not-MVCC, but have a suffix. For these the values
-		// useful for statically known exclusions to value separation. In
+		// must be stored with the key, since the concept of "older versions" is
-		// CockroachDB, this will be used for the lock table key space that has
+		// not defined. It is also useful for statically known exclusions to value
-		// non-empty suffixes, but those locks don't represent actual MVCC
+		// separation. In CockroachDB, this will be used for the lock table key
-		// versions (the suffix ordering is arbitrary). We will also need to add
+		// space that has non-empty suffixes, but those locks don't represent
-		// support for dynamically configured exclusions (we want the default to
+		// actual MVCC versions (the suffix ordering is arbitrary). We will also
-		// be to allow Pebble to decide whether to separate the value or not,
+		// need to add support for dynamically configured exclusions (we want the
-		// hence this is structured as exclusions), for example, for users of
+		// default to be to allow Pebble to decide whether to separate the value
-		// CockroachDB to dynamically exclude certain tables.
+		// or not, hence this is structured as exclusions), for example, for users
+		// of CockroachDB to dynamically exclude certain tables.
 		//
 		// Any change in exclusion behavior takes effect only on future written
 		// sstables, and does not start rewriting existing sstables.
+		//
+		// Even ignoring changes in this setting, exclusions are interpreted as a
+		// guidance by Pebble, and not necessarily honored. Specifically, user
+		// keys with multiple Pebble-versions *may* have the older versions stored
+		// in value blocks.
 		RequiredInPlaceValueBound UserKeyPrefixBound
 
 		// DisableIngestAsFlushable disables lazy ingestion of sstables through
@@ -626,15 +680,49 @@ type Options struct {
 		// major version is at least `FormatFlushableIngest`.
 		DisableIngestAsFlushable func() bool
 
-		// SharedStorage is a second FS-like storage medium that can be shared
+		// RemoteStorage enables use of remote storage (e.g. S3) for storing
-		// between multiple Pebble instances. It is used to store sstables only, and
+		// sstables. Setting this option enables use of CreateOnShared option and
-		// is managed by objstorage.Provider. Each sstable might only be written to
+		// allows ingestion of external files.
-		// by one Pebble instance, but other Pebble instances can possibly read the
+		RemoteStorage remote.StorageFactory
-		// same files if they have the path to get to them. The pebble instance that
+
-		// wrote a file should not delete it if other Pebble instances are known to
+		// If CreateOnShared is non-zero, new sstables are created on remote storage
-		// be reading this file. This FS is expected to have slower read/write
+		// (using CreateOnSharedLocator and with the appropriate
-		// performance than the default FS above.
+		// CreateOnSharedStrategy). These sstables can be shared between different
-		SharedStorage shared.Storage
+		// Pebble instances; the lifecycle of such objects is managed by the
+		// remote.Storage constructed by options.RemoteStorage.
+		//
+		// Can only be used when RemoteStorage is set (and recognizes
+		// CreateOnSharedLocator).
+		CreateOnShared        remote.CreateOnSharedStrategy
+		CreateOnSharedLocator remote.Locator
+
+		// CacheSizeBytesBytes is the size of the on-disk block cache for objects
+		// on shared storage in bytes. If it is 0, no cache is used.
+		SecondaryCacheSizeBytes int64
+
+		// IneffectualPointDeleteCallback is called in compactions/flushes if any
+		// single delete is being elided without deleting a point set/merge.
+		IneffectualSingleDeleteCallback func(userKey []byte)
+
+		// SingleDeleteInvariantViolationCallback is called in compactions/flushes if any
+		// single delete has consumed a Set/Merge, and there is another immediately older
+		// Set/SetWithDelete/Merge. The user of Pebble has violated the invariant under
+		// which SingleDelete can be used correctly.
+		//
+		// Consider the sequence SingleDelete#3, Set#2, Set#1. There are three
+		// ways some of these keys can first meet in a compaction.
+		//
+		// - All 3 keys in the same compaction: this callback will detect the
+		//   violation.
+		//
+		// - SingleDelete#3, Set#2 meet in a compaction first: Both keys will
+		//   disappear. The violation will not be detected, and the DB will have
+		//   Set#1 which is likely incorrect (from the user's perspective).
+		//
+		// - Set#2, Set#1 meet in a compaction first: The output will be Set#2,
+		//   which will later be consumed by SingleDelete#3. The violation will
+		//   not be detected and the DB will be correct.
+		SingleDeleteInvariantViolationCallback func(userKey []byte)
 	}
 
 	// Filters is a map from filter policy name to filter policy. It is used for
@@ -749,12 +837,18 @@ type Options struct {
 	// writing the contents of the old one in the
 	// background. MemTableStopWritesThreshold places a hard limit on the size of
 	// the queued MemTables.
-	MemTableSize int
+	//
+	// The default value is 4MB.
+	MemTableSize uint64
 
-	// Hard limit on the size of queued of MemTables. Writes are stopped when the
+	// Hard limit on the number of queued of MemTables. Writes are stopped when
-	// sum of the queued memtable sizes exceeds
+	// the sum of the queued memtable sizes exceeds:
-	// MemTableStopWritesThreshold*MemTableSize. This value should be at least 2
+	//   MemTableStopWritesThreshold * MemTableSize.
-	// or writes will stop whenever a MemTable is being flushed.
+	//
+	// This value should be at least 2 or writes will stop whenever a MemTable is
+	// being flushed.
+	//
+	// The default value is 2.
 	MemTableStopWritesThreshold int
 
 	// Merger defines the associative merge operation to use for merging values
@@ -839,6 +933,20 @@ type Options struct {
 	// changing options dynamically?
 	WALMinSyncInterval func() time.Duration
 
+	// TargetByteDeletionRate is the rate (in bytes per second) at which sstable file
+	// deletions are limited to (under normal circumstances).
+	//
+	// Deletion pacing is used to slow down deletions when compactions finish up
+	// or readers close and newly-obsolete files need cleaning up. Deleting lots
+	// of files at once can cause disk latency to go up on some SSDs, which this
+	// functionality guards against.
+	//
+	// This value is only a best-effort target; the effective rate can be
+	// higher if deletions are falling behind or disk space is running low.
+	//
+	// Setting this to 0 disables deletion pacing, which is also the default.
+	TargetByteDeletionRate int
+
 	// private options are only used by internal tests or are used internally
 	// for facilitating upgrade paths of unconfigurable functionality.
 	private struct {
@@ -872,6 +980,10 @@ type Options struct {
 		// A private option to disable stats collection.
 		disableTableStats bool
 
+		// testingAlwaysWaitForCleanup is set by some tests to force waiting for
+		// obsolete file deletion (to make events deterministic).
+		testingAlwaysWaitForCleanup bool
+
 		// fsCloser holds a closer that should be invoked after a DB using these
 		// Options is closed. This is used to automatically stop the
 		// long-running goroutine associated with the disk-health-checking FS.
@@ -983,7 +1095,7 @@ func (o *Options) EnsureDefaults() *Options {
 		o.MaxOpenFiles = 1000
 	}
 	if o.MemTableSize <= 0 {
-		o.MemTableSize = 4 << 20
+		o.MemTableSize = 4 << 20 // 4 MB
 	}
 	if o.MemTableStopWritesThreshold <= 0 {
 		o.MemTableStopWritesThreshold = 2
@@ -1024,8 +1136,8 @@ func (o *Options) EnsureDefaults() *Options {
 	if o.Experimental.CPUWorkPermissionGranter == nil {
 		o.Experimental.CPUWorkPermissionGranter = defaultCPUWorkGranter{}
 	}
-	if o.Experimental.MultiLevelCompactionHueristic == nil {
+	if o.Experimental.MultiLevelCompactionHeuristic == nil {
-		o.Experimental.MultiLevelCompactionHueristic = NoMultiLevel{}
+		o.Experimental.MultiLevelCompactionHeuristic = WriteAmpHeuristic{}
 	}
 
 	o.initMaps()
@@ -1144,7 +1256,7 @@ func (o *Options) String() string {
 	fmt.Fprintf(&buf, "  max_open_files=%d\n", o.MaxOpenFiles)
 	fmt.Fprintf(&buf, "  mem_table_size=%d\n", o.MemTableSize)
 	fmt.Fprintf(&buf, "  mem_table_stop_writes_threshold=%d\n", o.MemTableStopWritesThreshold)
-	fmt.Fprintf(&buf, "  min_deletion_rate=%d\n", o.Experimental.MinDeletionRate)
+	fmt.Fprintf(&buf, "  min_deletion_rate=%d\n", o.TargetByteDeletionRate)
 	fmt.Fprintf(&buf, "  merger=%s\n", o.Merger.Name)
 	fmt.Fprintf(&buf, "  read_compaction_rate=%d\n", o.Experimental.ReadCompactionRate)
 	fmt.Fprintf(&buf, "  read_sampling_multiplier=%d\n", o.Experimental.ReadSamplingMultiplier)
@@ -1165,6 +1277,8 @@ func (o *Options) String() string {
 	fmt.Fprintf(&buf, "  wal_bytes_per_sync=%d\n", o.WALBytesPerSync)
 	fmt.Fprintf(&buf, "  max_writer_concurrency=%d\n", o.Experimental.MaxWriterConcurrency)
 	fmt.Fprintf(&buf, "  force_writer_parallelism=%t\n", o.Experimental.ForceWriterParallelism)
+	fmt.Fprintf(&buf, "  secondary_cache_size_bytes=%d\n", o.Experimental.SecondaryCacheSizeBytes)
+	fmt.Fprintf(&buf, "  create_on_shared=%d\n", o.Experimental.CreateOnShared)
 
 	// Private options.
 	//
@@ -1322,7 +1436,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 					}
 				}
 			case "compaction_debt_concurrency":
-				o.Experimental.CompactionDebtConcurrency, err = strconv.Atoi(value)
+				o.Experimental.CompactionDebtConcurrency, err = strconv.ParseUint(value, 10, 64)
 			case "delete_range_flush_delay":
 				// NB: This is a deprecated serialization of the
 				// `flush_delay_delete_range`.
@@ -1354,7 +1468,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 				// version is valid right here.
 				var v uint64
 				v, err = strconv.ParseUint(value, 10, 64)
-				if vers := FormatMajorVersion(v); vers > FormatNewest || vers == FormatDefault {
+				if vers := FormatMajorVersion(v); vers > internalFormatNewest || vers == FormatDefault {
 					err = errors.Newf("unknown format major version %d", o.FormatMajorVersion)
 				}
 				if err == nil {
@@ -1387,14 +1501,14 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 			case "max_open_files":
 				o.MaxOpenFiles, err = strconv.Atoi(value)
 			case "mem_table_size":
-				o.MemTableSize, err = strconv.Atoi(value)
+				o.MemTableSize, err = strconv.ParseUint(value, 10, 64)
 			case "mem_table_stop_writes_threshold":
 				o.MemTableStopWritesThreshold, err = strconv.Atoi(value)
 			case "min_compaction_rate":
 				// Do nothing; option existed in older versions of pebble, and
 				// may be meaningful again eventually.
 			case "min_deletion_rate":
-				o.Experimental.MinDeletionRate, err = strconv.Atoi(value)
+				o.TargetByteDeletionRate, err = strconv.Atoi(value)
 			case "min_flush_rate":
 				// Do nothing; option existed in older versions of pebble, and
 				// may be meaningful again eventually.
@@ -1438,6 +1552,12 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 				o.Experimental.MaxWriterConcurrency, err = strconv.Atoi(value)
 			case "force_writer_parallelism":
 				o.Experimental.ForceWriterParallelism, err = strconv.ParseBool(value)
+			case "secondary_cache_size_bytes":
+				o.Experimental.SecondaryCacheSizeBytes, err = strconv.ParseInt(value, 10, 64)
+			case "create_on_shared":
+				var createOnSharedInt int64
+				createOnSharedInt, err = strconv.ParseInt(value, 10, 64)
+				o.Experimental.CreateOnShared = remote.CreateOnSharedStrategy(createOnSharedInt)
 			default:
 				if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key, value) {
 					return nil
@@ -1568,15 +1688,15 @@ func (o *Options) Validate() error {
 	}
 	if uint64(o.MemTableSize) >= maxMemTableSize {
 		fmt.Fprintf(&buf, "MemTableSize (%s) must be < %s\n",
-			humanize.Uint64(uint64(o.MemTableSize)), humanize.Uint64(maxMemTableSize))
+			humanize.Bytes.Uint64(uint64(o.MemTableSize)), humanize.Bytes.Uint64(maxMemTableSize))
 	}
 	if o.MemTableStopWritesThreshold < 2 {
 		fmt.Fprintf(&buf, "MemTableStopWritesThreshold (%d) must be >= 2\n",
 			o.MemTableStopWritesThreshold)
 	}
-	if o.FormatMajorVersion > FormatNewest {
+	if o.FormatMajorVersion > internalFormatNewest {
 		fmt.Fprintf(&buf, "FormatMajorVersion (%d) must be <= %d\n",
-			o.FormatMajorVersion, FormatNewest)
+			o.FormatMajorVersion, internalFormatNewest)
 	}
 	if o.TableCache != nil && o.Cache != o.TableCache.cache {
 		fmt.Fprintf(&buf, "underlying cache in the TableCache and the Cache dont match\n")
@@ -1596,6 +1716,7 @@ func (o *Options) MakeReaderOptions() sstable.ReaderOptions {
 		readerOpts.Comparer = o.Comparer
 		readerOpts.Filters = o.Filters
 		if o.Merger != nil {
+			readerOpts.Merge = o.Merger.Merge
 			readerOpts.MergerName = o.Merger.Name
 		}
 		readerOpts.LoggerAndTracer = o.LoggerAndTracer
@@ -1620,6 +1741,9 @@ func (o *Options) MakeWriterOptions(level int, format sstable.TableFormat) sstab
 	if format >= sstable.TableFormatPebblev3 {
 		writerOpts.ShortAttributeExtractor = o.Experimental.ShortAttributeExtractor
 		writerOpts.RequiredInPlaceValueBound = o.Experimental.RequiredInPlaceValueBound
+		if format >= sstable.TableFormatPebblev4 && level == numLevels-1 {
+			writerOpts.WritingToLowestLevel = true
+		}
 	}
 	levelOpts := o.Level(level)
 	writerOpts.BlockRestartInterval = levelOpts.BlockRestartInterval