Update dependencies

go.mod

@@ -1,10 +1,10 @@
 module main
 
-go 1.19
+go 1.22
 
 require (
-	github.com/cockroachdb/errors v1.10.0
+	github.com/cockroachdb/errors v1.11.3
-	github.com/cockroachdb/pebble v0.0.0-20230613192303-a8a7ebf58839
+	github.com/cockroachdb/pebble v1.1.1
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/hjson/hjson-go v3.3.0+incompatible
 	github.com/sirupsen/logrus v1.9.3
@@ -13,26 +13,27 @@ require (
 require (
 	github.com/DataDog/zstd v1.5.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cockroachdb/fifo v0.0.0-20240616162244-4768e80dfb9a // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/getsentry/sentry-go v0.21.0 // indirect
+	github.com/getsentry/sentry-go v0.28.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/klauspost/compress v1.16.6 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.15.1 // indirect
+	github.com/prometheus/client_golang v1.19.1 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
-	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
-	github.com/rogpeppe/go-internal v1.10.0 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
-	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
-	golang.org/x/sys v0.9.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
-	golang.org/x/text v0.10.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 )

go.sum

@@ -2,17 +2,22 @@ github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
 github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
-github.com/cockroachdb/errors v1.10.0 h1:lfxS8zZz1+OjtV4MtNWgboi/W5tyLEB6VQZBXN+0VUU=
+github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
-github.com/cockroachdb/errors v1.10.0/go.mod h1:lknhIsEVQ9Ss/qKDBQS/UqFSvPQjOwNq2qyKAxtHRqE=
+github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
+github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
+github.com/cockroachdb/fifo v0.0.0-20240616162244-4768e80dfb9a h1:f52TdbU4D5nozMAhO9TvTJ2ZMCXtN4VIAmfrrZ0JXQ4=
+github.com/cockroachdb/fifo v0.0.0-20240616162244-4768e80dfb9a/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20230613192303-a8a7ebf58839 h1:avSJHOeU0yPVq2rSBuBLlBC7Q6hU1J3IEPAt9fK7bvY=
+github.com/cockroachdb/pebble v1.1.1 h1:XnKU22oiCLy2Xn8vp1re67cXg4SAasg/WDt1NtcRFaw=
-github.com/cockroachdb/pebble v0.0.0-20230613192303-a8a7ebf58839/go.mod h1:TkdVsGYRqtULUppt2RbC+YaKtTHnHoWa2apfFrSKABw=
+github.com/cockroachdb/pebble v1.1.1/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
 github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -20,87 +25,91 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
-github.com/getsentry/sentry-go v0.21.0 h1:c9l5F1nPF30JIppulk4veau90PK6Smu3abgVtVQWon4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/getsentry/sentry-go v0.21.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
+github.com/getsentry/sentry-go v0.28.1 h1:zzaSm/vHmGllRM6Tpx1492r0YDzauArdBfkJRtY6P5k=
+github.com/getsentry/sentry-go v0.28.1/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hjson/hjson-go v3.3.0+incompatible h1:Rqr+Ya+0aCJMjaE4s8E9YKvuJLuLVpEvz4ONum52vnI=
 github.com/hjson/hjson-go v3.3.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
-github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
+github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
+github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
+github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
-github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
+github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
-github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
-github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
-github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
-github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
-github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
-github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
-github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
-github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
-golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
@@ -109,12 +118,13 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

vendor/github.com/cespare/xxhash/v2/README.md

@@ -70,3 +70,5 @@ benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$')
 - [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
 - [FreeCache](https://github.com/coocood/freecache)
 - [FastCache](https://github.com/VictoriaMetrics/fastcache)
+- [Ristretto](https://github.com/dgraph-io/ristretto)
+- [Badger](https://github.com/dgraph-io/badger)

vendor/github.com/cespare/xxhash/v2/xxhash.go

@@ -19,10 +19,13 @@ const (
 // Store the primes in an array as well.
 //
 // The consts are used when possible in Go code to avoid MOVs but we need a
-// contiguous array of the assembly code.
+// contiguous array for the assembly code.
 var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5}
 
 // Digest implements hash.Hash64.
+//
+// Note that a zero-valued Digest is not ready to receive writes.
+// Call Reset or create a Digest using New before calling other methods.
 type Digest struct {
 	v1    uint64
 	v2    uint64
@@ -33,19 +36,31 @@ type Digest struct {
 	n     int // how much of mem is used
 }
 
-// New creates a new Digest that computes the 64-bit xxHash algorithm.
+// New creates a new Digest with a zero seed.
 func New() *Digest {
+	return NewWithSeed(0)
+}
+
+// NewWithSeed creates a new Digest with the given seed.
+func NewWithSeed(seed uint64) *Digest {
 	var d Digest
-	d.Reset()
+	d.ResetWithSeed(seed)
 	return &d
 }
 
 // Reset clears the Digest's state so that it can be reused.
+// It uses a seed value of zero.
 func (d *Digest) Reset() {
-	d.v1 = primes[0] + prime2
+	d.ResetWithSeed(0)
-	d.v2 = prime2
+}
-	d.v3 = 0
+
-	d.v4 = -primes[0]
+// ResetWithSeed clears the Digest's state so that it can be reused.
+// It uses the given seed to initialize the state.
+func (d *Digest) ResetWithSeed(seed uint64) {
+	d.v1 = seed + prime1 + prime2
+	d.v2 = seed + prime2
+	d.v3 = seed
+	d.v4 = seed - prime1
 	d.total = 0
 	d.n = 0
 }

vendor/github.com/cespare/xxhash/v2/xxhash_asm.go

@@ -6,7 +6,7 @@
 
 package xxhash
 
-// Sum64 computes the 64-bit xxHash digest of b.
+// Sum64 computes the 64-bit xxHash digest of b with a zero seed.
 //
 //go:noescape
 func Sum64(b []byte) uint64

vendor/github.com/cespare/xxhash/v2/xxhash_other.go

@@ -3,7 +3,7 @@
 
 package xxhash
 
-// Sum64 computes the 64-bit xxHash digest of b.
+// Sum64 computes the 64-bit xxHash digest of b with a zero seed.
 func Sum64(b []byte) uint64 {
 	// A simpler version would be
 	//   d := New()

vendor/github.com/cespare/xxhash/v2/xxhash_safe.go

@@ -5,7 +5,7 @@
 
 package xxhash
 
-// Sum64String computes the 64-bit xxHash digest of s.
+// Sum64String computes the 64-bit xxHash digest of s with a zero seed.
 func Sum64String(s string) uint64 {
 	return Sum64([]byte(s))
 }

vendor/github.com/cespare/xxhash/v2/xxhash_unsafe.go

@@ -33,7 +33,7 @@ import (
 //
 // See https://github.com/golang/go/issues/42739 for discussion.
 
-// Sum64String computes the 64-bit xxHash digest of s.
+// Sum64String computes the 64-bit xxHash digest of s with a zero seed.
 // It may be faster than Sum64([]byte(s)) by avoiding a copy.
 func Sum64String(s string) uint64 {
 	b := *(*[]byte)(unsafe.Pointer(&sliceHeader{s, len(s)}))

vendor/github.com/cockroachdb/errors/README.md

@@ -209,12 +209,12 @@ return errors.Wrap(foo(), "foo")
   - how to access the detail: `Error()`, regular Go formatting, Sentry Report.
 
 - `WithDetail(error, string) error`, `WithDetailf(error, string, ...interface{}) error`, user-facing detail with contextual information.
-  - **when to use: need to embark a message string to output when the error is presented to a human.**
+  - **when to use: need to embark a message string to output when the error is presented to a developer.**
   - what it does: captures detail strings.
   - how to access the detail: `errors.GetAllDetails()`, `errors.FlattenDetails()` (all details are preserved), format with `%+v`. Not included in Sentry reports.
 
 - `WithHint(error, string) error`, `WithHintf(error, string, ...interface{}) error`: user-facing detail with suggestion for action to take.
-  - **when to use: need to embark a message string to output when the error is presented to a human.**
+  - **when to use: need to embark a message string to output when the error is presented to an end user.**
   - what it does: captures hint strings.
   - how to access the detail: `errors.GetAllHints()`, `errors.FlattenHints()` (hints are de-duplicated), format with `%+v`. Not included in Sentry reports.
 
@@ -536,7 +536,8 @@ Example use:
 | `WrapWithDepthf`                   | `WithMessagef` + `WithStackDepth`                                                 |
 | `AssertionFailedWithDepthf`        | `NewWithDepthf` + `WithAssertionFailure`                                          |
 | `NewAssertionErrorWithWrappedErrf` | `HandledWithMessagef` (barrier) + `WrapWithDepthf` +  `WithAssertionFailure`      |
-
+| `Join`                             | `JoinWithDepth` (see below)                                                       |
+| `JoinWithDepth`                    | multi-cause wrapper + `WithStackDepth`                                            |
 ## API (not constructing error objects)
 
 The following is a summary of the non-constructor API functions, grouped by category.
@@ -573,10 +574,16 @@ func RegisterLeafDecoder(typeName TypeKey, decoder LeafDecoder)
 func RegisterLeafEncoder(typeName TypeKey, encoder LeafEncoder)
 func RegisterWrapperDecoder(typeName TypeKey, decoder WrapperDecoder)
 func RegisterWrapperEncoder(typeName TypeKey, encoder WrapperEncoder)
+func RegisterWrapperEncoderWithMessageOverride (typeName TypeKey, encoder WrapperEncoderWithMessageOverride)
+func RegisterMultiCauseEncoder(theType TypeKey, encoder MultiCauseEncoder)
+func RegisterMultiCauseDecoder(theType TypeKey, decoder MultiCauseDecoder)
 type LeafEncoder = func(ctx context.Context, err error) (msg string, safeDetails []string, payload proto.Message)
 type LeafDecoder = func(ctx context.Context, msg string, safeDetails []string, payload proto.Message) error
 type WrapperEncoder = func(ctx context.Context, err error) (msgPrefix string, safeDetails []string, payload proto.Message)
+type WrapperEncoderWithMessageOverride = func(ctx context.Context, err error) (msgPrefix string, safeDetails []string, payload proto.Message, overrideError bool)
 type WrapperDecoder = func(ctx context.Context, cause error, msgPrefix string, safeDetails []string, payload proto.Message) error
+type MultiCauseEncoder = func(ctx context.Context, err error) (msg string, safeDetails []string, payload proto.Message)
+type MultiCauseDecoder = func(ctx context.Context, causes []error, msgPrefix string, safeDetails []string, payload proto.Message) error
 
 // Registering package renames for custom error types.
 func RegisterTypeMigration(previousPkgPath, previousTypeName string, newType error)

vendor/github.com/cockroachdb/errors/contexttags/contexttags.go

@@ -25,7 +25,7 @@ import (
 // WithContextTags captures the k/v pairs stored in the context via the
 // `logtags` package and annotates them on the error.
 //
-// Only the stromg representation of values remains available. This is
+// Only the strong representation of values remains available. This is
 // because the library cannot guarantee that the underlying value is
 // preserved across the network. To avoid creating a stateful interface
 // (where the user code needs to know whether an error has traveled

vendor/github.com/cockroachdb/errors/contexttags_api.go

@@ -24,7 +24,7 @@ import (
 // WithContextTags captures the k/v pairs stored in the context via the
 // `logtags` package and annotates them on the error.
 //
-// Only the stromg representation of values remains available. This is
+// Only the strong representation of values remains available. This is
 // because the library cannot guarantee that the underlying value is
 // preserved across the network. To avoid creating a stateful interface
 // (where the user code needs to know whether an error has traveled

vendor/github.com/cockroachdb/errors/errbase/decode.go

@@ -57,6 +57,15 @@ func decodeLeaf(ctx context.Context, enc *errorspb.EncodedErrorLeaf) error {
 			return genErr
 		}
 		// Decoding failed, we'll drop through to opaqueLeaf{} below.
+	} else if decoder, ok := multiCauseDecoders[typeKey]; ok {
+		causes := make([]error, len(enc.MultierrorCauses))
+		for i, e := range enc.MultierrorCauses {
+			causes[i] = DecodeError(ctx, *e)
+		}
+		genErr := decoder(ctx, causes, enc.Message, enc.Details.ReportablePayload, payload)
+		if genErr != nil {
+			return genErr
+		}
 	} else {
 		// Shortcut for non-registered proto-encodable error types:
 		// if it already implements `error`, it's good to go.
@@ -66,6 +75,19 @@ func decodeLeaf(ctx context.Context, enc *errorspb.EncodedErrorLeaf) error {
 		}
 	}
 
+	if len(enc.MultierrorCauses) > 0 {
+		causes := make([]error, len(enc.MultierrorCauses))
+		for i, e := range enc.MultierrorCauses {
+			causes[i] = DecodeError(ctx, *e)
+		}
+		leaf := &opaqueLeafCauses{
+			causes: causes,
+		}
+		leaf.msg = enc.Message
+		leaf.details = enc.Details
+		return leaf
+	}
+
 	// No decoder and no error type: we'll keep what we received and
 	// make it ready to re-encode exactly (if the error leaves over the
 	// network again).
@@ -97,7 +119,7 @@ func decodeWrapper(ctx context.Context, enc *errorspb.EncodedWrapper) error {
 	typeKey := TypeKey(enc.Details.ErrorTypeMark.FamilyName)
 	if decoder, ok := decoders[typeKey]; ok {
 		// Yes, use it.
-		genErr := decoder(ctx, cause, enc.MessagePrefix, enc.Details.ReportablePayload, payload)
+		genErr := decoder(ctx, cause, enc.Message, enc.Details.ReportablePayload, payload)
 		if genErr != nil {
 			// Decoding succeeded. Use this.
 			return genErr
@@ -107,9 +129,10 @@ func decodeWrapper(ctx context.Context, enc *errorspb.EncodedWrapper) error {
 
 	// Otherwise, preserve all details about the original object.
 	return &opaqueWrapper{
-		cause:   cause,
+		cause:       cause,
-		prefix:  enc.MessagePrefix,
+		prefix:      enc.Message,
-		details: enc.Details,
+		details:     enc.Details,
+		messageType: MessageType(enc.MessageType),
 	}
 }
 
@@ -160,3 +183,24 @@ type WrapperDecoder = func(ctx context.Context, cause error, msgPrefix string, s
 
 // registry for RegisterWrapperType.
 var decoders = map[TypeKey]WrapperDecoder{}
+
+// MultiCauseDecoder is to be provided (via RegisterMultiCauseDecoder
+// above) by additional multi-cause wrapper types not yet known by the
+// library. A nil return indicates that decoding was not successful.
+type MultiCauseDecoder = func(ctx context.Context, causes []error, msgPrefix string, safeDetails []string, payload proto.Message) error
+
+// registry for RegisterMultiCauseDecoder.
+var multiCauseDecoders = map[TypeKey]MultiCauseDecoder{}
+
+// RegisterMultiCauseDecoder can be used to register new multi-cause
+// wrapper types to the library. Registered wrappers will be decoded
+// using their own Go type when an error is decoded. Multi-cause
+// wrappers that have not been registered will be decoded using the
+// opaqueWrapper type.
+func RegisterMultiCauseDecoder(theType TypeKey, decoder MultiCauseDecoder) {
+	if decoder == nil {
+		delete(multiCauseDecoders, theType)
+	} else {
+		multiCauseDecoders[theType] = decoder
+	}
+}

vendor/github.com/cockroachdb/errors/errbase/encode.go

@@ -33,18 +33,24 @@ func EncodeError(ctx context.Context, err error) EncodedError {
 	if cause := UnwrapOnce(err); cause != nil {
 		return encodeWrapper(ctx, err, cause)
 	}
-	// Not a causer.
+	return encodeLeaf(ctx, err, UnwrapMulti(err))
-	return encodeLeaf(ctx, err)
 }
 
-// encodeLeaf encodes a leaf error.
+// encodeLeaf encodes a leaf error. This function accepts a `causes`
-func encodeLeaf(ctx context.Context, err error) EncodedError {
+// argument because we encode multi-cause errors using the Leaf
+// protobuf. This was done to enable backwards compatibility when
+// introducing this functionality since the Wrapper type already has a
+// required single `cause` field.
+func encodeLeaf(ctx context.Context, err error, causes []error) EncodedError {
 	var msg string
 	var details errorspb.EncodedErrorDetails
 
 	if e, ok := err.(*opaqueLeaf); ok {
 		msg = e.msg
 		details = e.details
+	} else if e, ok := err.(*opaqueLeafCauses); ok {
+		msg = e.msg
+		details = e.details
 	} else {
 		details.OriginalTypeName, details.ErrorTypeMark.FamilyName, details.ErrorTypeMark.Extension = getTypeDetails(err, false /*onlyFamily*/)
 
@@ -74,11 +80,21 @@ func encodeLeaf(ctx context.Context, err error) EncodedError {
 		details.FullDetails = encodeAsAny(ctx, err, payload)
 	}
 
+	var cs []*EncodedError
+	if len(causes) > 0 {
+		cs = make([]*EncodedError, len(causes))
+		for i, ee := range causes {
+			ee := EncodeError(ctx, ee)
+			cs[i] = &ee
+		}
+	}
+
 	return EncodedError{
 		Error: &errorspb.EncodedError_Leaf{
 			Leaf: &errorspb.EncodedErrorLeaf{
-				Message: msg,
+				Message:          msg,
-				Details: details,
+				Details:          details,
+				MultierrorCauses: cs,
 			},
 		},
 	}
@@ -115,10 +131,18 @@ func encodeAsAny(ctx context.Context, err error, payload proto.Message) *types.A
 func encodeWrapper(ctx context.Context, err, cause error) EncodedError {
 	var msg string
 	var details errorspb.EncodedErrorDetails
+	messageType := Prefix
 
 	if e, ok := err.(*opaqueWrapper); ok {
+		// We delegate all knowledge of the error string
+		// to the original encoder and do not try to re-engineer
+		// the prefix out of the error. This helps maintain
+		// backward compatibility with earlier versions of the
+		// encoder which don't have any understanding of
+		// error string ownership by the wrapper.
 		msg = e.prefix
 		details = e.details
+		messageType = e.messageType
 	} else {
 		details.OriginalTypeName, details.ErrorTypeMark.FamilyName, details.ErrorTypeMark.Extension = getTypeDetails(err, false /*onlyFamily*/)
 
@@ -127,12 +151,12 @@ func encodeWrapper(ctx context.Context, err, cause error) EncodedError {
 		// If we have a manually registered encoder, use that.
 		typeKey := TypeKey(details.ErrorTypeMark.FamilyName)
 		if enc, ok := encoders[typeKey]; ok {
-			msg, details.ReportablePayload, payload = enc(ctx, err)
+			msg, details.ReportablePayload, payload, messageType = enc(ctx, err)
 		} else {
 			// No encoder.
 			// In that case, we'll try to compute a message prefix
 			// manually.
-			msg = extractPrefix(err, cause)
+			msg, messageType = extractPrefix(err, cause)
 
 			// If there are known safe details, use them.
 			if s, ok := err.(SafeDetailer); ok {
@@ -148,9 +172,10 @@ func encodeWrapper(ctx context.Context, err, cause error) EncodedError {
 	return EncodedError{
 		Error: &errorspb.EncodedError_Wrapper{
 			Wrapper: &errorspb.EncodedWrapper{
-				Cause:         EncodeError(ctx, cause),
+				Cause:       EncodeError(ctx, cause),
-				MessagePrefix: msg,
+				Message:     msg,
-				Details:       details,
+				Details:     details,
+				MessageType: errorspb.MessageType(messageType),
 			},
 		},
 	}
@@ -158,21 +183,36 @@ func encodeWrapper(ctx context.Context, err, cause error) EncodedError {
 
 // extractPrefix extracts the prefix from a wrapper's error message.
 // For example,
-//    err := errors.New("bar")
+//
-//    err = errors.Wrap(err, "foo")
+//	err := errors.New("bar")
-//    extractPrefix(err)
+//	err = errors.Wrap(err, "foo")
+//	extractPrefix(err)
+//
 // returns "foo".
-func extractPrefix(err, cause error) string {
+//
+// If a presumed wrapper does not have a message prefix, it is assumed
+// to override the entire error message and `extractPrefix` returns
+// the entire message and the boolean `true` to signify that the causes
+// should not be appended to it.
+func extractPrefix(err, cause error) (string, MessageType) {
 	causeSuffix := cause.Error()
 	errMsg := err.Error()
 
 	if strings.HasSuffix(errMsg, causeSuffix) {
 		prefix := errMsg[:len(errMsg)-len(causeSuffix)]
+		// If error msg matches exactly then this is a wrapper
+		// with no message of its own.
+		if len(prefix) == 0 {
+			return "", Prefix
+		}
 		if strings.HasSuffix(prefix, ": ") {
-			return prefix[:len(prefix)-2]
+			return prefix[:len(prefix)-2], Prefix
 		}
 	}
-	return ""
+	// If we don't have the cause as a suffix, then we have
+	// some other string as our error msg, preserve that and
+	// mark as override
+	return errMsg, FullMessage
 }
 
 func getTypeDetails(
@@ -183,6 +223,8 @@ func getTypeDetails(
 	switch t := err.(type) {
 	case *opaqueLeaf:
 		return t.details.OriginalTypeName, t.details.ErrorTypeMark.FamilyName, t.details.ErrorTypeMark.Extension
+	case *opaqueLeafCauses:
+		return t.details.OriginalTypeName, t.details.ErrorTypeMark.FamilyName, t.details.ErrorTypeMark.Extension
 	case *opaqueWrapper:
 		return t.details.OriginalTypeName, t.details.ErrorTypeMark.FamilyName, t.details.ErrorTypeMark.Extension
 	}
@@ -286,6 +328,28 @@ type LeafEncoder = func(ctx context.Context, err error) (msg string, safeDetails
 // registry for RegisterLeafEncoder.
 var leafEncoders = map[TypeKey]LeafEncoder{}
 
+// RegisterMultiCauseEncoder can be used to register new multi-cause
+// error types to the library. Registered types will be encoded using
+// their own Go type when an error is encoded. Multi-cause wrappers
+// that have not been registered will be encoded using the
+// opaqueWrapper type.
+func RegisterMultiCauseEncoder(theType TypeKey, encoder MultiCauseEncoder) {
+	// This implementation is a simple wrapper around `LeafEncoder`
+	// because we implemented multi-cause error wrapper encoding into a
+	// `Leaf` instead of a `Wrapper` for smoother backwards
+	// compatibility support. Exposing this detail to consumers of the
+	// API is confusing and hence avoided. The causes of the error are
+	// encoded separately regardless of this encoder's implementation.
+	RegisterLeafEncoder(theType, encoder)
+}
+
+// MultiCauseEncoder is to be provided (via RegisterMultiCauseEncoder
+// above) by additional multi-cause wrapper types not yet known to this
+// library. The encoder will automatically extract and encode the
+// causes of this error by calling `Unwrap()` and expecting a slice of
+// errors.
+type MultiCauseEncoder = func(ctx context.Context, err error) (msg string, safeDetails []string, payload proto.Message)
+
 // RegisterWrapperEncoder can be used to register new wrapper types to
 // the library. Registered wrappers will be encoded using their own
 // Go type when an error is encoded. Wrappers that have not been
@@ -295,6 +359,35 @@ var leafEncoders = map[TypeKey]LeafEncoder{}
 // or a different type, ensure that RegisterTypeMigration() was called
 // prior to RegisterWrapperEncoder().
 func RegisterWrapperEncoder(theType TypeKey, encoder WrapperEncoder) {
+	RegisterWrapperEncoderWithMessageType(
+		theType,
+		func(ctx context.Context, err error) (
+			msgPrefix string,
+			safeDetails []string,
+			payload proto.Message,
+			messageType MessageType,
+		) {
+			prefix, details, payload := encoder(ctx, err)
+			return prefix, details, payload, messageType
+		})
+}
+
+// RegisterWrapperEncoderWithMessageType can be used to register
+// new wrapper types to the library. Registered wrappers will be
+// encoded using their own Go type when an error is encoded. Wrappers
+// that have not been registered will be encoded using the
+// opaqueWrapper type.
+//
+// This function differs from RegisterWrapperEncoder by allowing the
+// caller to explicitly decide whether the wrapper owns the entire
+// error message or not. Otherwise, the relationship is inferred.
+//
+// Note: if the error type has been migrated from a previous location
+// or a different type, ensure that RegisterTypeMigration() was called
+// prior to RegisterWrapperEncoder().
+func RegisterWrapperEncoderWithMessageType(
+	theType TypeKey, encoder WrapperEncoderWithMessageType,
+) {
 	if encoder == nil {
 		delete(encoders, theType)
 	} else {
@@ -304,7 +397,42 @@ func RegisterWrapperEncoder(theType TypeKey, encoder WrapperEncoder) {
 
 // WrapperEncoder is to be provided (via RegisterWrapperEncoder above)
 // by additional wrapper types not yet known to this library.
-type WrapperEncoder = func(ctx context.Context, err error) (msgPrefix string, safeDetails []string, payload proto.Message)
+type WrapperEncoder func(ctx context.Context, err error) (
+	msgPrefix string,
+	safeDetails []string,
+	payload proto.Message,
+)
+
+// MessageType is used to encode information about an error message
+// within a wrapper error type. This information is used to affect
+// display logic.
+type MessageType errorspb.MessageType
+
+// Values below should match the ones in errorspb.MessageType for
+// direct conversion.
+const (
+	// Prefix denotes an error message that should be prepended to the
+	// message of its cause.
+	Prefix MessageType = MessageType(errorspb.MessageType_PREFIX)
+	// FullMessage denotes an error message that contains the text of its
+	// causes and can be displayed standalone.
+	FullMessage = MessageType(errorspb.MessageType_FULL_MESSAGE)
+)
+
+// WrapperEncoderWithMessageType is to be provided (via
+// RegisterWrapperEncoderWithMessageType above) by additional wrapper
+// types not yet known to this library. This encoder returns an
+// additional enum which indicates whether the wrapper owns the error
+// message completely instead of simply being a prefix with the error
+// message of its causes appended to it. This information is encoded
+// along with the prefix in order to provide context during error
+// display.
+type WrapperEncoderWithMessageType func(ctx context.Context, err error) (
+	msgPrefix string,
+	safeDetails []string,
+	payload proto.Message,
+	messageType MessageType,
+)
 
 // registry for RegisterWrapperType.
-var encoders = map[TypeKey]WrapperEncoder{}
+var encoders = map[TypeKey]WrapperEncoderWithMessageType{}

vendor/github.com/cockroachdb/errors/errbase/format_error.go

@@ -102,7 +102,13 @@ func formatErrorInternal(err error, s fmt.State, verb rune, redactableOutput boo
 		// to enable stack trace de-duplication. This requires a
 		// post-order traversal. Since we have a linked list, the best we
 		// can do is a recursion.
-		p.formatRecursive(err, true /* isOutermost */, true /* withDetail */)
+		p.formatRecursive(
+			err,
+			true,  /* isOutermost */
+			true,  /* withDetail */
+			false, /* withDepth */
+			0,     /* depth */
+		)
 
 		// We now have all the data, we can render the result.
 		p.formatEntries(err)
@@ -146,7 +152,13 @@ func formatErrorInternal(err error, s fmt.State, verb rune, redactableOutput boo
 		// by calling FormatError(), in which case we'd get an infinite
 		// recursion. So we have no choice but to peel the data
 		// and then assemble the pieces ourselves.
-		p.formatRecursive(err, true /* isOutermost */, false /* withDetail */)
+		p.formatRecursive(
+			err,
+			true,  /* isOutermost */
+			false, /* withDetail */
+			false, /* withDepth */
+			0,     /* depth */
+		)
 		p.formatSingleLineOutput()
 		p.finishDisplay(verb)
 
@@ -195,7 +207,19 @@ func (s *state) formatEntries(err error) {
 	// Wraps: (N) <details>
 	//
 	for i, j := len(s.entries)-2, 2; i >= 0; i, j = i-1, j+1 {
-		fmt.Fprintf(&s.finalBuf, "\nWraps: (%d)", j)
+		s.finalBuf.WriteByte('\n')
+		// Extra indentation starts at depth==2 because the direct
+		// children of the root error area already printed on separate
+		// newlines.
+		for m := 0; m < s.entries[i].depth-1; m += 1 {
+			if m == s.entries[i].depth-2 {
+				s.finalBuf.WriteString("└─ ")
+			} else {
+				s.finalBuf.WriteByte(' ')
+				s.finalBuf.WriteByte(' ')
+			}
+		}
+		fmt.Fprintf(&s.finalBuf, "Wraps: (%d)", j)
 		entry := s.entries[i]
 		s.printEntry(entry)
 	}
@@ -278,8 +302,9 @@ func (s *state) printEntry(entry formatEntry) {
 //
 // This function is used both when FormatError() is called indirectly
 // from .Error(), e.g. in:
-//      (e *myType) Error() { return fmt.Sprintf("%v", e) }
+//
-//      (e *myType) Format(s fmt.State, verb rune) { errors.FormatError(s, verb, e) }
+//	(e *myType) Error() { return fmt.Sprintf("%v", e) }
+//	(e *myType) Format(s fmt.State, verb rune) { errors.FormatError(s, verb, e) }
 //
 // and also to print the first line in the output of a %+v format.
 //
@@ -329,13 +354,35 @@ func (s *state) formatSingleLineOutput() {
 // s.finalBuf is untouched. The conversion of s.entries
 // to s.finalBuf is done by formatSingleLineOutput() and/or
 // formatEntries().
-func (s *state) formatRecursive(err error, isOutermost, withDetail bool) {
+//
+// `withDepth` and `depth` are used to tag subtrees of multi-cause
+// errors for added indentation during printing. Once a multi-cause
+// error is encountered, all subsequent calls with set `withDepth` to
+// true, and increment `depth` during recursion. This information is
+// persisted into the generated entries and used later to display the
+// error with increased indentation based in the depth.
+func (s *state) formatRecursive(err error, isOutermost, withDetail, withDepth bool, depth int) int {
 	cause := UnwrapOnce(err)
+	numChildren := 0
 	if cause != nil {
-		// Recurse first.
+		// Recurse first, which populates entries list starting from innermost
-		s.formatRecursive(cause, false /*isOutermost*/, withDetail)
+		// entry. If we've previously seen a multi-cause wrapper, `withDepth`
+		// will be true, and we'll record the depth below ensuring that extra
+		// indentation is applied to this inner cause during printing.
+		// Otherwise, we maintain "straight" vertical formatting by keeping the
+		// parent callers `withDepth` value of `false` by default.
+		numChildren += s.formatRecursive(cause, false, withDetail, withDepth, depth+1)
 	}
 
+	causes := UnwrapMulti(err)
+	for _, c := range causes {
+		// Override `withDepth` to true for all child entries ensuring they have
+		// indentation applied during formatting to distinguish them from
+		// parents.
+		numChildren += s.formatRecursive(c, false, withDetail, true, depth+1)
+	}
+	// inserted := len(s.entries) - 1 - startChildren
+
 	// Reinitialize the state for this stage of wrapping.
 	s.wantDetail = withDetail
 	s.needSpace = false
@@ -354,11 +401,11 @@ func (s *state) formatRecursive(err error, isOutermost, withDetail bool) {
 		bufIsRedactable = true
 		desiredShortening := v.SafeFormatError((*safePrinter)(s))
 		if desiredShortening == nil {
-			// The error wants to elide the short messages from inner
+			// The error wants to elide the short messages from inner causes.
-			// causes. Do it.
+			// Read backwards through list of entries up to the number of new
-			for i := range s.entries {
+			// entries created "under" this one amount and mark `elideShort`
-				s.entries[i].elideShort = true
+			// true.
-			}
+			s.elideShortChildren(numChildren)
 		}
 
 	case Formatter:
@@ -366,9 +413,7 @@ func (s *state) formatRecursive(err error, isOutermost, withDetail bool) {
 		if desiredShortening == nil {
 			// The error wants to elide the short messages from inner
 			// causes. Do it.
-			for i := range s.entries {
+			s.elideShortChildren(numChildren)
-				s.entries[i].elideShort = true
-			}
 		}
 
 	case fmt.Formatter:
@@ -389,7 +434,11 @@ func (s *state) formatRecursive(err error, isOutermost, withDetail bool) {
 				s.lastStack = st.StackTrace()
 			}
 		} else {
-			s.formatSimple(err, cause)
+			if elideCauseMsg := s.formatSimple(err, cause); elideCauseMsg {
+				// The error wants to elide the short messages from inner
+				// causes. Do it.
+				s.elideShortChildren(numChildren)
+			}
 		}
 
 	default:
@@ -411,9 +460,7 @@ func (s *state) formatRecursive(err error, isOutermost, withDetail bool) {
 				if desiredShortening == nil {
 					// The error wants to elide the short messages from inner
 					// causes. Do it.
-					for i := range s.entries {
+					s.elideShortChildren(numChildren)
-						s.entries[i].elideShort = true
-					}
 				}
 				break
 			}
@@ -422,12 +469,21 @@ func (s *state) formatRecursive(err error, isOutermost, withDetail bool) {
 			// If the error did not implement errors.Formatter nor
 			// fmt.Formatter, but it is a wrapper, still attempt best effort:
 			// print what we can at this level.
-			s.formatSimple(err, cause)
+			elideChildren := s.formatSimple(err, cause)
+			// always elideChildren when dealing with multi-cause errors.
+			if len(causes) > 0 {
+				elideChildren = true
+			}
+			if elideChildren {
+				// The error wants to elide the short messages from inner
+				// causes. Do it.
+				s.elideShortChildren(numChildren)
+			}
 		}
 	}
 
 	// Collect the result.
-	entry := s.collectEntry(err, bufIsRedactable)
+	entry := s.collectEntry(err, bufIsRedactable, withDepth, depth)
 
 	// If there's an embedded stack trace, also collect it.
 	// This will get either a stack from pkg/errors, or ours.
@@ -441,9 +497,22 @@ func (s *state) formatRecursive(err error, isOutermost, withDetail bool) {
 	// Remember the entry for later rendering.
 	s.entries = append(s.entries, entry)
 	s.buf = bytes.Buffer{}
+
+	return numChildren + 1
 }
 
-func (s *state) collectEntry(err error, bufIsRedactable bool) formatEntry {
+// elideShortChildren takes a number of entries to set `elideShort` to
+// false. The reason a number of entries is needed is that we may be
+// eliding a subtree of causes in the case of a multi-cause error. In
+// the multi-cause case, we need to know how many of the prior errors
+// in the list of entries is a child of this subtree.
+func (s *state) elideShortChildren(newEntries int) {
+	for i := 0; i < newEntries; i++ {
+		s.entries[len(s.entries)-1-i].elideShort = true
+	}
+}
+
+func (s *state) collectEntry(err error, bufIsRedactable bool, withDepth bool, depth int) formatEntry {
 	entry := formatEntry{err: err}
 	if s.wantDetail {
 		// The buffer has been populated as a result of formatting with
@@ -480,6 +549,10 @@ func (s *state) collectEntry(err error, bufIsRedactable bool) formatEntry {
 		}
 	}
 
+	if withDepth {
+		entry.depth = depth
+	}
+
 	return entry
 }
 
@@ -500,16 +573,23 @@ func RegisterSpecialCasePrinter(fn safeErrorPrinterFn) {
 // formatSimple performs a best effort at extracting the details at a
 // given level of wrapping when the error object does not implement
 // the Formatter interface.
-func (s *state) formatSimple(err, cause error) {
+// Returns true if we want to elide errors from causal chain.
+func (s *state) formatSimple(err, cause error) bool {
 	var pref string
+	elideCauses := false
 	if cause != nil {
-		pref = extractPrefix(err, cause)
+		var messageType MessageType
+		pref, messageType = extractPrefix(err, cause)
+		if messageType == FullMessage {
+			elideCauses = true
+		}
 	} else {
 		pref = err.Error()
 	}
 	if len(pref) > 0 {
 		s.Write([]byte(pref))
 	}
+	return elideCauses
 }
 
 // finishDisplay renders s.finalBuf into s.State.
@@ -568,7 +648,7 @@ type state struct {
 
 	// entries collect the result of formatRecursive(). They are
 	// consumed by formatSingleLineOutput() and formatEntries() to
-	// procude the contents of finalBuf.
+	// produce the contents of finalBuf.
 	entries []formatEntry
 
 	// buf collects the details of the current error object at a given
@@ -690,6 +770,11 @@ type formatEntry struct {
 	// truncated to avoid duplication of entries. This is used to
 	// display a truncation indicator during verbose rendering.
 	elidedStackTrace bool
+
+	// depth, if positive, represents a nesting depth of this error as
+	// a causer of others. This is used with verbose printing to
+	// illustrate the nesting depth for multi-cause error wrappers.
+	depth int
 }
 
 // String is used for debugging only.
@@ -711,6 +796,12 @@ func (s *state) Write(b []byte) (n int, err error) {
 
 	for i, c := range b {
 		if c == '\n' {
+			//if s.needNewline > 0 {
+			//	for i := 0; i < s.needNewline-1; i++ {
+			//		s.buf.Write(detailSep[:len(sep)-1])
+			//	}
+			//	s.needNewline = 0
+			//}
 			// Flush all the bytes seen so far.
 			s.buf.Write(b[k:i])
 			// Don't print the newline itself; instead, prepare the state so
@@ -740,6 +831,11 @@ func (s *state) Write(b []byte) (n int, err error) {
 			s.notEmpty = true
 		}
 	}
+	//if s.needNewline > 0 {
+	//	for i := 0; i < s.needNewline-1; i++ {
+	//		s.buf.Write(detailSep[:len(sep)-1])
+	//	}
+	//}
 	s.buf.Write(b[k:])
 	return len(b), nil
 }
@@ -766,6 +862,9 @@ func (p *state) switchOver() {
 	p.buf = bytes.Buffer{}
 	p.notEmpty = false
 	p.hasDetail = true
+
+	// One of the newlines is accounted for in the switch over.
+	// p.needNewline -= 1
 }
 
 func (s *printer) Detail() bool {
@@ -829,7 +928,7 @@ func (s *safePrinter) enhanceArgs(args []interface{}) {
 			args[i] = redact.Safe(thisStack)
 			lastSeen = st
 		}
-		// In contrast with (*printer).enhanceArgs(), we dont use a
+		// In contrast with (*printer).enhanceArgs(), we don't use a
 		// special case for `error` here, because the redact package
 		// already helps us recursing into a safe print for
 		// error objects.

vendor/github.com/cockroachdb/errors/errbase/format_simple.go

@@ -1,36 +0,0 @@
-// Copyright 2019 The Cockroach Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-// implied. See the License for the specific language governing
-// permissions and limitations under the License.
-
-package errbase
-
-import "io"
-
-// formatSimpleError is a helper used by FormatError() for the top
-// level error/wrapper argument, if it does not implement the
-// errors.Formatter interface.
-func formatSimpleError(err error, p *state, sep string) error {
-	if cause := UnwrapOnce(err); cause != nil {
-		pref := extractPrefix(err, cause)
-		p.buf.WriteString(pref)
-		if pref != "" {
-			p.buf.WriteByte(':')
-			p.buf.WriteString(sep)
-		}
-		err = cause
-	} else {
-		io.WriteString(&p.buf, err.Error())
-		err = nil
-	}
-	return err
-}

vendor/github.com/cockroachdb/errors/errbase/opaque.go

@@ -30,19 +30,39 @@ type opaqueLeaf struct {
 	details errorspb.EncodedErrorDetails
 }
 
+// opaqueLeafCauses is used when receiving an unknown multi-cause
+// wrapper type. Its important property is that if it is communicated
+// back to some network system that _does_ know about the type, the
+// original object can be restored. We encode multi-cause errors as
+// leaf nodes over the network, in order to support backwards
+// compatibility with existing single-cause wrapper messages.
+//
+// This struct *must* be initialized with a non-nil causes value in
+// order to comply with go stdlib expectations for `Unwrap()`.
+type opaqueLeafCauses struct {
+	opaqueLeaf
+	causes []error
+}
+
 var _ error = (*opaqueLeaf)(nil)
 var _ SafeDetailer = (*opaqueLeaf)(nil)
 var _ fmt.Formatter = (*opaqueLeaf)(nil)
 var _ SafeFormatter = (*opaqueLeaf)(nil)
 
+var _ error = (*opaqueLeafCauses)(nil)
+var _ SafeDetailer = (*opaqueLeafCauses)(nil)
+var _ fmt.Formatter = (*opaqueLeafCauses)(nil)
+var _ SafeFormatter = (*opaqueLeafCauses)(nil)
+
 // opaqueWrapper is used when receiving an unknown wrapper type.
 // Its important property is that if it is communicated
 // back to some network system that _does_ know about
 // the type, the original object can be restored.
 type opaqueWrapper struct {
-	cause   error
+	cause       error
-	prefix  string
+	prefix      string
-	details errorspb.EncodedErrorDetails
+	details     errorspb.EncodedErrorDetails
+	messageType MessageType
 }
 
 var _ error = (*opaqueWrapper)(nil)
@@ -53,6 +73,9 @@ var _ SafeFormatter = (*opaqueWrapper)(nil)
 func (e *opaqueLeaf) Error() string { return e.msg }
 
 func (e *opaqueWrapper) Error() string {
+	if e.messageType == FullMessage {
+		return e.prefix
+	}
 	if e.prefix == "" {
 		return e.cause.Error()
 	}
@@ -66,8 +89,12 @@ func (e *opaqueWrapper) Unwrap() error { return e.cause }
 func (e *opaqueLeaf) SafeDetails() []string    { return e.details.ReportablePayload }
 func (e *opaqueWrapper) SafeDetails() []string { return e.details.ReportablePayload }
 
-func (e *opaqueLeaf) Format(s fmt.State, verb rune)    { FormatError(e, s, verb) }
+func (e *opaqueLeaf) Format(s fmt.State, verb rune)       { FormatError(e, s, verb) }
-func (e *opaqueWrapper) Format(s fmt.State, verb rune) { FormatError(e, s, verb) }
+func (e *opaqueLeafCauses) Format(s fmt.State, verb rune) { FormatError(e, s, verb) }
+func (e *opaqueWrapper) Format(s fmt.State, verb rune)    { FormatError(e, s, verb) }
+
+// opaqueLeafCauses is a multi-cause wrapper
+func (e *opaqueLeafCauses) Unwrap() []error { return e.causes }
 
 func (e *opaqueLeaf) SafeFormatError(p Printer) (next error) {
 	p.Print(e.msg)
@@ -103,5 +130,8 @@ func (e *opaqueWrapper) SafeFormatError(p Printer) (next error) {
 			p.Printf("\npayload type: %s", redact.Safe(e.details.FullDetails.TypeUrl))
 		}
 	}
+	if e.messageType == FullMessage {
+		return nil
+	}
 	return e.cause
 }

vendor/github.com/cockroachdb/errors/errbase/unwrap.go

@@ -27,6 +27,16 @@ package errbase
 // It supports both errors implementing causer (`Cause()` method, from
 // github.com/pkg/errors) and `Wrapper` (`Unwrap()` method, from the
 // Go 2 error proposal).
+//
+// UnwrapOnce treats multi-errors (those implementing the
+// `Unwrap() []error` interface as leaf-nodes since they cannot
+// reasonably be iterated through to a single cause. These errors
+// are typically constructed as a result of `fmt.Errorf` which results
+// in a `wrapErrors` instance that contains an interpolated error
+// string along with a list of causes.
+//
+// The go stdlib does not define output on `Unwrap()` for a multi-cause
+// error, so we default to nil here.
 func UnwrapOnce(err error) (cause error) {
 	switch e := err.(type) {
 	case interface{ Cause() error }:
@@ -39,6 +49,7 @@ func UnwrapOnce(err error) (cause error) {
 
 // UnwrapAll accesses the root cause object of the error.
 // If the error has no cause (leaf error), it is returned directly.
+// UnwrapAll treats multi-errors as leaf nodes.
 func UnwrapAll(err error) error {
 	for {
 		if cause := UnwrapOnce(err); cause != nil {
@@ -49,3 +60,12 @@ func UnwrapAll(err error) error {
 	}
 	return err
 }
+
+// UnwrapMulti access the slice of causes that an error contains, if it is a
+// multi-error.
+func UnwrapMulti(err error) []error {
+	if me, ok := err.(interface{ Unwrap() []error }); ok {
+		return me.Unwrap()
+	}
+	return nil
+}

vendor/github.com/cockroachdb/errors/errbase_api.go

@@ -86,6 +86,20 @@ func GetTypeKey(err error) TypeKey { return errbase.GetTypeKey(err) }
 // A nil return indicates that decoding was not successful.
 type LeafDecoder = errbase.LeafDecoder
 
+// MultiCauseDecoder is to be provided (via RegisterMultiCauseDecoder
+// above) by additional multi-cause wrapper types not yet known by the
+// library. A nil return indicates that decoding was not successful.
+type MultiCauseDecoder = errbase.MultiCauseDecoder
+
+// RegisterMultiCauseDecoder can be used to register new multi-cause
+// wrapper types to the library. Registered wrappers will be decoded
+// using their own Go type when an error is decoded. Multi-cause
+// wrappers that have not been registered will be decoded using the
+// opaqueWrapper type.
+func RegisterMultiCauseDecoder(theType TypeKey, decoder MultiCauseDecoder) {
+	errbase.RegisterMultiCauseDecoder(theType, decoder)
+}
+
 // RegisterWrapperDecoder can be used to register new wrapper types to
 // the library. Registered wrappers will be decoded using their own
 // Go type when an error is decoded. Wrappers that have not been
@@ -135,6 +149,41 @@ func RegisterWrapperEncoder(typeName TypeKey, encoder WrapperEncoder) {
 // by additional wrapper types not yet known to this library.
 type WrapperEncoder = errbase.WrapperEncoder
 
+// RegisterWrapperEncoderWithMessageType can be used to register new wrapper
+// types to the library. These wrappers can optionally override the child error
+// messages with their own error string instead of relying on iterative
+// concatenation. Registered wrappers will be encoded using their own Go type
+// when an error is encoded. Wrappers that have not been registered will be
+// encoded using the opaqueWrapper type.
+//
+// Note: if the error type has been migrated from a previous location
+// or a different type, ensure that RegisterTypeMigration() was called
+// prior to RegisterWrapperEncoder().
+func RegisterWrapperEncoderWithMessageType(typeName TypeKey, encoder WrapperEncoderWithMessageType) {
+	errbase.RegisterWrapperEncoderWithMessageType(typeName, encoder)
+}
+
+// WrapperEncoderWithMessageType is to be provided (via
+// RegisterWrapperEncoderWithMessageType) by additional wrapper
+// types not yet known to this library.
+type WrapperEncoderWithMessageType = errbase.WrapperEncoderWithMessageType
+
+// RegisterMultiCauseEncoder can be used to register new multi-cause
+// error types to the library. Registered types will be encoded using
+// their own Go type when an error is encoded. Multi-cause wrappers
+// that have not been registered will be encoded using the
+// opaqueWrapper type.
+func RegisterMultiCauseEncoder(typeName TypeKey, encoder MultiCauseEncoder) {
+	errbase.RegisterMultiCauseEncoder(typeName, encoder)
+}
+
+// MultiCauseEncoder is to be provided (via RegisterMultiCauseEncoder
+// above) by additional multi-cause wrapper types not yet known to this
+// library. The encoder will automatically extract and encode the
+// causes of this error by calling `Unwrap()` and expecting a slice of
+// errors.
+type MultiCauseEncoder = errbase.MultiCauseEncoder
+
 // SetWarningFn enables configuration of the warning function.
 func SetWarningFn(fn func(context.Context, string, ...interface{})) { errbase.SetWarningFn(fn) }
 
@@ -185,7 +234,8 @@ func Formattable(err error) fmt.Formatter { return errbase.Formattable(err) }
 // The value of previousTypeName must be the result of calling
 // reflect.TypeOf(err).String() on the original error object.
 // This is usually composed as follows:
-//     [*]<shortpackage>.<errortype>
+//
+//	[*]<shortpackage>.<errortype>
 //
 // For example, Go's standard error type has name "*errors.errorString".
 // The asterisk indicates that `errorString` implements the `error`

vendor/github.com/cockroachdb/errors/errorspb/errors.pb.go

@@ -24,10 +24,42 @@ var _ = math.Inf
 // proto package needs to be updated.
 const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
 
+// MessageType encodes information regarding the interpretation of an
+// error's message string.
+type MessageType int32
+
+const (
+	// Keeping zero value as default for backwards compatibility.
+	// PREFIX is a message that should be prepended to its cause.
+	MessageType_PREFIX MessageType = 0
+	// FULL_MESSAGE is a complete error message that can be displayed
+	// without its cause.
+	MessageType_FULL_MESSAGE MessageType = 1
+)
+
+var MessageType_name = map[int32]string{
+	0: "PREFIX",
+	1: "FULL_MESSAGE",
+}
+
+var MessageType_value = map[string]int32{
+	"PREFIX":       0,
+	"FULL_MESSAGE": 1,
+}
+
+func (x MessageType) String() string {
+	return proto.EnumName(MessageType_name, int32(x))
+}
+
+func (MessageType) EnumDescriptor() ([]byte, []int) {
+	return fileDescriptor_ddc818d0729874b8, []int{0}
+}
+
 // EncodedError is the wire-encodable representation
 // of an error (or error cause chain).
 type EncodedError struct {
 	// Types that are valid to be assigned to Error:
+	//
 	//	*EncodedError_Leaf
 	//	*EncodedError_Wrapper
 	Error isEncodedError_Error `protobuf_oneof:"error"`
@@ -108,7 +140,7 @@ func (*EncodedError) XXX_OneofWrappers() []interface{} {
 }
 
 // EncodedErrorLeaf is the wire-encodable representation
-// of an error leaf.
+// of an error leaf or a multi-cause wrapper from go 1.20+
 type EncodedErrorLeaf struct {
 	// The main error message (mandatory), that can be printed to human
 	// users and may contain PII. This contains the value of the leaf
@@ -116,6 +148,11 @@ type EncodedErrorLeaf struct {
 	Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"`
 	// The error details.
 	Details EncodedErrorDetails `protobuf:"bytes,2,opt,name=details,proto3" json:"details"`
+	// multierror_causes is a list of errors that contain the causal tree
+	// of this leaf. If this field is not empty, then this leaf encodes
+	// an error from go 1.20 or later that encodes multiple causes in its
+	// chain.
+	MultierrorCauses []*EncodedError `protobuf:"bytes,3,rep,name=multierror_causes,json=multierrorCauses,proto3" json:"multierror_causes,omitempty"`
 }
 
 func (m *EncodedErrorLeaf) Reset()         { *m = EncodedErrorLeaf{} }
@@ -210,17 +247,23 @@ var xxx_messageInfo_EncodedErrorDetails proto.InternalMessageInfo
 type EncodedWrapper struct {
 	// The cause error. Mandatory.
 	Cause EncodedError `protobuf:"bytes,1,opt,name=cause,proto3" json:"cause"`
-	// The wrapper message prefix (which may be empty). This
+	// The wrapper message. This could either be a full error message
-	// isbprinted before the cause's own message when
+	// that can be printed independently, or a (potentially empty) prefix
-	// constructing a full message. This may contain PII.
+	// which is printed before the cause's own message to construct the
+	// full message. This may contain PII.
 	//
 	// This is extracted automatically:
 	//
 	// - for wrappers that have a registered encoder,
 	// - otherwise, when the wrapper's Error() has its cause's Error() as suffix.
-	MessagePrefix string `protobuf:"bytes,2,opt,name=message_prefix,json=messagePrefix,proto3" json:"message_prefix,omitempty"`
+	Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
 	// The error details.
 	Details EncodedErrorDetails `protobuf:"bytes,3,opt,name=details,proto3" json:"details"`
+	// message_type encodes the interpretation of `message`. Prior
+	// versions will not set this field and it will be left as `PREFIX`.
+	// This retains backwards compatibility since the new behavior is
+	// only enabled when this enum is set to `FULL_MESSAGE`.
+	MessageType MessageType `protobuf:"varint,4,opt,name=message_type,json=messageType,proto3,enum=cockroach.errorspb.MessageType" json:"message_type,omitempty"`
 }
 
 func (m *EncodedWrapper) Reset()         { *m = EncodedWrapper{} }
@@ -382,6 +425,7 @@ func (m *ErrnoPayload) XXX_DiscardUnknown() {
 var xxx_messageInfo_ErrnoPayload proto.InternalMessageInfo
 
 func init() {
+	proto.RegisterEnum("cockroach.errorspb.MessageType", MessageType_name, MessageType_value)
 	proto.RegisterType((*EncodedError)(nil), "cockroach.errorspb.EncodedError")
 	proto.RegisterType((*EncodedErrorLeaf)(nil), "cockroach.errorspb.EncodedErrorLeaf")
 	proto.RegisterType((*EncodedErrorDetails)(nil), "cockroach.errorspb.EncodedErrorDetails")
@@ -394,46 +438,51 @@ func init() {
 func init() { proto.RegisterFile("errorspb/errors.proto", fileDescriptor_ddc818d0729874b8) }
 
 var fileDescriptor_ddc818d0729874b8 = []byte{
-	// 624 bytes of a gzipped FileDescriptorProto
+	// 693 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x53, 0x41, 0x6f, 0xd3, 0x4c,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x53, 0x41, 0x4f, 0x13, 0x41,
-	0x10, 0xb5, 0xbf, 0xa4, 0x4d, 0x32, 0x49, 0xfa, 0x95, 0xa5, 0x48, 0x6e, 0x45, 0xdd, 0x34, 0x80,
+	0x14, 0xde, 0xa5, 0x85, 0xb6, 0xaf, 0x05, 0xcb, 0x88, 0xc9, 0x42, 0x64, 0x29, 0xab, 0x89, 0x04,
-	0xa8, 0x2a, 0x70, 0x24, 0x38, 0x20, 0x21, 0x84, 0x44, 0x45, 0x45, 0x0f, 0x50, 0x2a, 0x53, 0x09,
+	0x75, 0x9b, 0xe0, 0xc1, 0xc4, 0x18, 0x13, 0xaa, 0x05, 0x4c, 0x28, 0x92, 0x05, 0xa3, 0xf1, 0xb2,
-	0x89, 0x8b, 0xb5, 0x49, 0xc6, 0xee, 0xaa, 0xb6, 0xd7, 0xda, 0x75, 0x44, 0xfd, 0x2f, 0x90, 0xf8,
+	0x99, 0xb6, 0xd3, 0x65, 0xc2, 0xee, 0xce, 0x66, 0x66, 0x1b, 0xd9, 0x7f, 0x61, 0xe2, 0x5f, 0xf2,
-	0x3f, 0x9c, 0x7b, 0xec, 0xb1, 0x27, 0x04, 0xa9, 0xf8, 0x17, 0x1c, 0x90, 0x77, 0xbd, 0x6d, 0x0a,
+	0xc0, 0x91, 0x23, 0x27, 0xa3, 0x25, 0xfe, 0x04, 0x6f, 0x1e, 0xcc, 0xce, 0xee, 0xd8, 0xa2, 0x28,
-	0x15, 0x3d, 0x70, 0x1b, 0xbf, 0x79, 0xe3, 0xf7, 0x3c, 0x6f, 0x0c, 0xb7, 0x50, 0x08, 0x2e, 0x64,
+	0x07, 0x6f, 0x6f, 0xde, 0xfb, 0xbe, 0xf7, 0xde, 0xf7, 0xde, 0x1b, 0xb8, 0x45, 0x38, 0x67, 0x5c,
-	0x36, 0x1c, 0xe8, 0xc2, 0xcb, 0x04, 0xcf, 0x39, 0x21, 0x23, 0x3e, 0x3a, 0x14, 0x9c, 0x8e, 0x0e,
+	0x44, 0xdd, 0x66, 0x66, 0xd8, 0x11, 0x67, 0x31, 0x43, 0xa8, 0xc7, 0x7a, 0xc7, 0x9c, 0xe1, 0xde,
-	0x3c, 0x43, 0x58, 0x59, 0x8e, 0x38, 0x8f, 0x62, 0x1c, 0x28, 0xc6, 0x70, 0x12, 0x0e, 0x68, 0x5a,
+	0x91, 0xad, 0x00, 0x4b, 0x8b, 0x1e, 0x63, 0x9e, 0x4f, 0x9a, 0x12, 0xd1, 0x1d, 0x0e, 0x9a, 0x38,
-	0x68, 0xfa, 0xca, 0x52, 0xc4, 0x23, 0xae, 0xca, 0x41, 0x59, 0x69, 0xb4, 0xff, 0xd9, 0x86, 0xce,
+	0x4c, 0x32, 0xf8, 0xd2, 0x82, 0xc7, 0x3c, 0x26, 0xcd, 0x66, 0x6a, 0x65, 0x5e, 0xeb, 0xa3, 0x0e,
-	0x76, 0x3a, 0xe2, 0x63, 0x1c, 0x6f, 0x97, 0x2f, 0x21, 0x4f, 0xa1, 0x1e, 0x23, 0x0d, 0x1d, 0xbb,
+	0xb5, 0x76, 0xd8, 0x63, 0x7d, 0xd2, 0x6f, 0xa7, 0x49, 0xd0, 0x13, 0x28, 0xfa, 0x04, 0x0f, 0x0c,
-	0x67, 0x6f, 0xb4, 0x1f, 0xdd, 0xf5, 0xfe, 0x14, 0xf1, 0x66, 0xf9, 0xaf, 0x91, 0x86, 0x3b, 0x96,
+	0xbd, 0xa1, 0xaf, 0x55, 0x37, 0xee, 0xda, 0x7f, 0x16, 0xb1, 0x27, 0xf1, 0xbb, 0x04, 0x0f, 0x76,
-	0xaf, 0x66, 0xc8, 0x73, 0x68, 0x7c, 0x14, 0x34, 0xcb, 0x50, 0x38, 0xff, 0xa9, 0xf1, 0xfe, 0x5f,
+	0x34, 0x47, 0x72, 0xd0, 0x33, 0x28, 0xbd, 0xe7, 0x38, 0x8a, 0x08, 0x37, 0xa6, 0x24, 0xdd, 0xfa,
-	0xc6, 0xdf, 0x6b, 0xe6, 0x8e, 0xe5, 0x9b, 0xa1, 0xad, 0x06, 0xcc, 0x29, 0x56, 0x7f, 0x02, 0x8b,
+	0x07, 0xfd, 0x4d, 0x86, 0xdc, 0xd1, 0x1c, 0x45, 0x6a, 0x95, 0x60, 0x5a, 0xa2, 0xac, 0x4f, 0x3a,
-	0xbf, 0x8b, 0x10, 0x07, 0x1a, 0x09, 0x4a, 0x49, 0x23, 0x54, 0xde, 0x5a, 0xbe, 0x79, 0x24, 0xaf,
+	0xd4, 0x7f, 0xaf, 0x82, 0x0c, 0x28, 0x05, 0x44, 0x08, 0xec, 0x11, 0xd9, 0x5c, 0xc5, 0x51, 0x4f,
-	0xa0, 0x31, 0xc6, 0x9c, 0xb2, 0x58, 0x56, 0xb2, 0xf7, 0xaf, 0x73, 0xfd, 0x52, 0xd3, 0xb7, 0xea,
+	0xb4, 0x0d, 0xa5, 0x3e, 0x89, 0x31, 0xf5, 0x45, 0x5e, 0xf7, 0xde, 0x75, 0x6d, 0xbf, 0xc8, 0xe0,
-	0xc7, 0x5f, 0xd7, 0x2c, 0xdf, 0x4c, 0xf7, 0x7f, 0xda, 0x70, 0xf3, 0x0a, 0x1a, 0x79, 0x00, 0x84,
+	0xad, 0xe2, 0xe9, 0xe7, 0x15, 0xcd, 0x51, 0x6c, 0xd4, 0x81, 0xf9, 0x60, 0xe8, 0xc7, 0x54, 0x72,
-	0x0b, 0x16, 0xb1, 0x94, 0xc6, 0x41, 0x5e, 0x64, 0x18, 0xa4, 0x34, 0x31, 0x2e, 0x16, 0x4d, 0x67,
+	0xdc, 0x1e, 0x1e, 0x0a, 0x22, 0x8c, 0x42, 0xa3, 0xb0, 0x56, 0xdd, 0x68, 0x5c, 0x97, 0xd2, 0xa9,
-	0xbf, 0xc8, 0x70, 0x97, 0x26, 0x48, 0xde, 0xc2, 0xff, 0x4a, 0x54, 0x53, 0x13, 0x2a, 0x0e, 0x2b,
+	0x8f, 0xa9, 0xcf, 0x25, 0xd3, 0xfa, 0xa1, 0xc3, 0xcd, 0x2b, 0xaa, 0xa2, 0x07, 0x80, 0x18, 0xa7,
-	0x5b, 0xeb, 0x57, 0xda, 0x2a, 0x8b, 0x72, 0xf6, 0x0d, 0x15, 0x87, 0x95, 0xa1, 0x2e, 0xce, 0x82,
+	0x1e, 0x0d, 0xb1, 0xef, 0xc6, 0x49, 0x44, 0xdc, 0x10, 0x07, 0x4a, 0x54, 0x5d, 0x45, 0x0e, 0x93,
-	0xe4, 0x21, 0x10, 0x81, 0x19, 0x17, 0x39, 0x1d, 0xc6, 0x18, 0x64, 0xb4, 0x88, 0x39, 0x1d, 0x3b,
+	0x88, 0xec, 0xe1, 0x80, 0xa0, 0x57, 0x70, 0x23, 0xeb, 0x47, 0x42, 0x03, 0xcc, 0x8f, 0x73, 0x95,
-	0xb5, 0x5e, 0x6d, 0xa3, 0xe5, 0xdf, 0xb8, 0xe8, 0xec, 0xe9, 0x06, 0x79, 0x02, 0x9d, 0x70, 0x12,
+	0xab, 0x57, 0xb6, 0x94, 0x1a, 0x29, 0xb7, 0x83, 0xf9, 0x71, 0xae, 0x6f, 0x96, 0x4c, 0x3a, 0xd1,
-	0xc7, 0x81, 0xd9, 0x49, 0x5d, 0x89, 0x2f, 0x79, 0xfa, 0x34, 0x3c, 0x73, 0x1a, 0xde, 0x8b, 0xb4,
+	0x43, 0x40, 0x9c, 0x44, 0x8c, 0xc7, 0xb8, 0xeb, 0x13, 0x37, 0xc2, 0x89, 0xcf, 0x70, 0x5f, 0xca,
-	0xf0, 0xdb, 0x25, 0xb3, 0xfa, 0xcc, 0xfe, 0x17, 0x1b, 0x16, 0x2e, 0x87, 0x43, 0x9e, 0xc1, 0xdc,
+	0xac, 0x38, 0xf3, 0xe3, 0xc8, 0x7e, 0x16, 0x40, 0x8f, 0xa1, 0x36, 0x18, 0xfa, 0xbe, 0xab, 0x46,
-	0x88, 0x4e, 0x24, 0x56, 0xe7, 0xd0, 0xbb, 0x6e, 0xb1, 0xd5, 0x07, 0xe8, 0x21, 0x72, 0x0f, 0x16,
+	0x5c, 0x94, 0xc5, 0x17, 0xec, 0xec, 0xd4, 0x6c, 0x75, 0x6a, 0xf6, 0x66, 0x98, 0x38, 0xd5, 0x14,
-	0xaa, 0x8c, 0x82, 0x4c, 0x60, 0xc8, 0x8e, 0xd4, 0x22, 0x5a, 0x7e, 0xb7, 0x42, 0xf7, 0x14, 0x38,
+	0x99, 0xcb, 0xb4, 0xbe, 0xeb, 0x30, 0x77, 0x79, 0xd9, 0xe8, 0x29, 0x4c, 0xcb, 0xa9, 0xe6, 0xe7,
-	0x9b, 0x5f, 0xed, 0x9f, 0xf2, 0xdb, 0x85, 0xee, 0xa5, 0x75, 0x92, 0x35, 0x68, 0x87, 0x34, 0x61,
+	0x75, 0xed, 0x50, 0x73, 0x01, 0x19, 0x69, 0xf2, 0x02, 0xa6, 0xfe, 0x7a, 0x01, 0x85, 0xff, 0xba,
-	0x71, 0x31, 0x9b, 0x18, 0x68, 0x48, 0x65, 0x75, 0x1b, 0x5a, 0x78, 0x94, 0x63, 0x2a, 0x19, 0x4f,
+	0x80, 0x16, 0xd4, 0xf2, 0x9c, 0x72, 0xdc, 0x52, 0xec, 0xdc, 0xc6, 0xca, 0x55, 0xd9, 0x3a, 0x19,
-	0x2b, 0x73, 0x17, 0x40, 0x7f, 0x13, 0x16, 0xde, 0xe5, 0x82, 0xa5, 0x91, 0x34, 0xbb, 0x75, 0x2e,
+	0x2e, 0x1d, 0xab, 0x53, 0x0d, 0xc6, 0x0f, 0x6b, 0x0f, 0x66, 0x2f, 0x6d, 0x01, 0xad, 0x40, 0x75,
-	0xac, 0xda, 0x6a, 0xff, 0xe7, 0xda, 0x3f, 0xca, 0x1f, 0x49, 0x88, 0x94, 0x1b, 0xea, 0x2a, 0x40,
+	0x80, 0x03, 0xea, 0x27, 0x93, 0x8b, 0x86, 0xcc, 0x25, 0x57, 0x7c, 0x1b, 0x2a, 0xe4, 0x24, 0x26,
-	0x79, 0x1a, 0x01, 0x96, 0xa0, 0x92, 0xae, 0xf9, 0xad, 0x12, 0x51, 0x2c, 0x42, 0xa0, 0x4e, 0xc5,
+	0xa1, 0xa0, 0x2c, 0xcc, 0xa5, 0x8d, 0x1d, 0xd6, 0x3a, 0xcc, 0x1d, 0xc4, 0x9c, 0x86, 0x9e, 0x50,
-	0xe8, 0xa0, 0x12, 0x55, 0x35, 0xb9, 0x03, 0x5d, 0x26, 0x83, 0x0c, 0x45, 0xc2, 0xa4, 0x72, 0x54,
+	0x2b, 0x31, 0xc6, 0x72, 0x75, 0xb9, 0x36, 0xf5, 0xb4, 0xbe, 0xa5, 0xff, 0x99, 0xf3, 0x90, 0x29,
-	0xae, 0xa3, 0xe9, 0x77, 0x98, 0xdc, 0x3b, 0xc7, 0xc8, 0x32, 0x34, 0x99, 0x0c, 0xf0, 0x88, 0xc9,
+	0xe8, 0x32, 0x40, 0x7a, 0x51, 0x2e, 0x49, 0x9d, 0xb2, 0x74, 0xc1, 0xa9, 0xa4, 0x1e, 0x89, 0x42,
-	0x5c, 0x45, 0xdb, 0xf4, 0x1b, 0x4c, 0x6e, 0x97, 0x8f, 0xa4, 0x07, 0x1d, 0x26, 0x83, 0x94, 0xe7,
+	0x08, 0x8a, 0x98, 0xf7, 0x8e, 0xf2, 0xa2, 0xd2, 0x46, 0x77, 0x60, 0x96, 0x0a, 0x37, 0x22, 0x3c,
-	0x55, 0x7b, 0x4e, 0xb5, 0x81, 0xc9, 0x5d, 0x9e, 0x6b, 0xc6, 0x2a, 0x00, 0x93, 0x41, 0xce, 0x12,
+	0xa0, 0x42, 0x76, 0x94, 0x8e, 0xb4, 0xec, 0xd4, 0xa8, 0xd8, 0xff, 0xe5, 0x43, 0x8b, 0x50, 0xa6,
-	0xe4, 0x93, 0xdc, 0x99, 0x57, 0xfd, 0x16, 0x93, 0xfb, 0x1a, 0x20, 0xeb, 0xea, 0x05, 0x39, 0x26,
+	0xc2, 0x25, 0x27, 0x54, 0xc4, 0x72, 0x48, 0x65, 0xa7, 0x44, 0x45, 0x3b, 0x7d, 0xa2, 0x06, 0xd4,
-	0x19, 0x17, 0x54, 0x14, 0x4e, 0x43, 0x11, 0xda, 0x4c, 0xee, 0x1b, 0x68, 0x6b, 0xf3, 0xf8, 0xbb,
+	0xa8, 0x70, 0x43, 0x16, 0xe7, 0xe1, 0x69, 0x19, 0x06, 0x2a, 0xf6, 0x58, 0x9c, 0x21, 0x96, 0x01,
-	0x6b, 0x1d, 0x4f, 0x5d, 0xfb, 0x64, 0xea, 0xda, 0xa7, 0x53, 0xd7, 0xfe, 0x36, 0x75, 0xed, 0x4f,
+	0xa8, 0x70, 0x63, 0x1a, 0x10, 0x36, 0x8c, 0x8d, 0x19, 0x19, 0xaf, 0x50, 0x71, 0x98, 0x39, 0xd0,
-	0x67, 0xae, 0x75, 0x72, 0xe6, 0x5a, 0xa7, 0x67, 0xae, 0xf5, 0xa1, 0x69, 0x22, 0x1b, 0xce, 0xab,
+	0xaa, 0x4c, 0x10, 0x93, 0x20, 0x62, 0x1c, 0xf3, 0xc4, 0x28, 0x49, 0x40, 0x95, 0x8a, 0x43, 0xe5,
-	0x5b, 0x7b, 0xfc, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x63, 0x5c, 0xa2, 0x72, 0xc1, 0x04, 0x00, 0x00,
+	0x5a, 0xbf, 0x0f, 0xd5, 0x89, 0xf9, 0x23, 0x80, 0x99, 0x7d, 0xa7, 0xbd, 0xf5, 0xf2, 0x6d, 0x5d,
+	0x43, 0x75, 0xa8, 0x6d, 0xbd, 0xde, 0xdd, 0x75, 0x3b, 0xed, 0x83, 0x83, 0xcd, 0xed, 0x76, 0x5d,
+	0x6f, 0xad, 0x9f, 0x7e, 0x35, 0xb5, 0xd3, 0x91, 0xa9, 0x9f, 0x8d, 0x4c, 0xfd, 0x7c, 0x64, 0xea,
+	0x5f, 0x46, 0xa6, 0xfe, 0xe1, 0xc2, 0xd4, 0xce, 0x2e, 0x4c, 0xed, 0xfc, 0xc2, 0xd4, 0xde, 0x95,
+	0xd5, 0x56, 0xbb, 0x33, 0xf2, 0x9e, 0x1f, 0xfd, 0x0c, 0x00, 0x00, 0xff, 0xff, 0xd7, 0x2c, 0x21,
+	0xe3, 0x75, 0x05, 0x00, 0x00,
 }
 
 func (m *EncodedError) Marshal() (dAtA []byte, err error) {
@@ -530,6 +579,20 @@ func (m *EncodedErrorLeaf) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	_ = i
 	var l int
 	_ = l
+	if len(m.MultierrorCauses) > 0 {
+		for iNdEx := len(m.MultierrorCauses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.MultierrorCauses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintErrors(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x1a
+		}
+	}
 	{
 		size, err := m.Details.MarshalToSizedBuffer(dAtA[:i])
 		if err != nil {
@@ -631,6 +694,11 @@ func (m *EncodedWrapper) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	_ = i
 	var l int
 	_ = l
+	if m.MessageType != 0 {
+		i = encodeVarintErrors(dAtA, i, uint64(m.MessageType))
+		i--
+		dAtA[i] = 0x20
+	}
 	{
 		size, err := m.Details.MarshalToSizedBuffer(dAtA[:i])
 		if err != nil {
@@ -641,10 +709,10 @@ func (m *EncodedWrapper) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	}
 	i--
 	dAtA[i] = 0x1a
-	if len(m.MessagePrefix) > 0 {
+	if len(m.Message) > 0 {
-		i -= len(m.MessagePrefix)
+		i -= len(m.Message)
-		copy(dAtA[i:], m.MessagePrefix)
+		copy(dAtA[i:], m.Message)
-		i = encodeVarintErrors(dAtA, i, uint64(len(m.MessagePrefix)))
+		i = encodeVarintErrors(dAtA, i, uint64(len(m.Message)))
 		i--
 		dAtA[i] = 0x12
 	}
@@ -874,6 +942,12 @@ func (m *EncodedErrorLeaf) Size() (n int) {
 	}
 	l = m.Details.Size()
 	n += 1 + l + sovErrors(uint64(l))
+	if len(m.MultierrorCauses) > 0 {
+		for _, e := range m.MultierrorCauses {
+			l = e.Size()
+			n += 1 + l + sovErrors(uint64(l))
+		}
+	}
 	return n
 }
 
@@ -910,12 +984,15 @@ func (m *EncodedWrapper) Size() (n int) {
 	_ = l
 	l = m.Cause.Size()
 	n += 1 + l + sovErrors(uint64(l))
-	l = len(m.MessagePrefix)
+	l = len(m.Message)
 	if l > 0 {
 		n += 1 + l + sovErrors(uint64(l))
 	}
 	l = m.Details.Size()
 	n += 1 + l + sovErrors(uint64(l))
+	if m.MessageType != 0 {
+		n += 1 + sovErrors(uint64(m.MessageType))
+	}
 	return n
 }
 
@@ -1202,6 +1279,40 @@ func (m *EncodedErrorLeaf) Unmarshal(dAtA []byte) error {
 				return err
 			}
 			iNdEx = postIndex
+		case 3:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field MultierrorCauses", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowErrors
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthErrors
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthErrors
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.MultierrorCauses = append(m.MultierrorCauses, &EncodedError{})
+			if err := m.MultierrorCauses[len(m.MultierrorCauses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipErrors(dAtA[iNdEx:])
@@ -1470,7 +1581,7 @@ func (m *EncodedWrapper) Unmarshal(dAtA []byte) error {
 			iNdEx = postIndex
 		case 2:
 			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field MessagePrefix", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType)
 			}
 			var stringLen uint64
 			for shift := uint(0); ; shift += 7 {
@@ -1498,7 +1609,7 @@ func (m *EncodedWrapper) Unmarshal(dAtA []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.MessagePrefix = string(dAtA[iNdEx:postIndex])
+			m.Message = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
 		case 3:
 			if wireType != 2 {
@@ -1533,6 +1644,25 @@ func (m *EncodedWrapper) Unmarshal(dAtA []byte) error {
 				return err
 			}
 			iNdEx = postIndex
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field MessageType", wireType)
+			}
+			m.MessageType = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowErrors
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.MessageType |= MessageType(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
 		default:
 			iNdEx = preIndex
 			skippy, err := skipErrors(dAtA[iNdEx:])

vendor/github.com/cockroachdb/errors/errorspb/errors.proto

@@ -17,7 +17,7 @@ message EncodedError {
 }
 
 // EncodedErrorLeaf is the wire-encodable representation
-// of an error leaf.
+// of an error leaf or a multi-cause wrapper from go 1.20+
 message EncodedErrorLeaf {
   // The main error message (mandatory), that can be printed to human
   // users and may contain PII. This contains the value of the leaf
@@ -26,6 +26,12 @@ message EncodedErrorLeaf {
 
   // The error details.
   EncodedErrorDetails details = 2 [(gogoproto.nullable) = false];
+
+  // multierror_causes is a list of errors that contain the causal tree
+  // of this leaf. If this field is not empty, then this leaf encodes
+  // an error from go 1.20 or later that encodes multiple causes in its
+  // chain.
+  repeated EncodedError multierror_causes = 3;
 }
 
 message EncodedErrorDetails {
@@ -60,24 +66,42 @@ message EncodedErrorDetails {
   google.protobuf.Any full_details = 4;
 }
 
+// MessageType encodes information regarding the interpretation of an
+// error's message string.
+enum MessageType {
+  // Keeping zero value as default for backwards compatibility.
+  // PREFIX is a message that should be prepended to its cause.
+  PREFIX = 0;
+  // FULL_MESSAGE is a complete error message that can be displayed
+  // without its cause.
+  FULL_MESSAGE = 1;
+}
+
 // EncodedWrapper is the wire-encodable representation
 // of an error wrapper.
 message EncodedWrapper {
   // The cause error. Mandatory.
   EncodedError cause = 1 [(gogoproto.nullable) = false];
 
-  // The wrapper message prefix (which may be empty). This
+  // The wrapper message. This could either be a full error message
-  // isbprinted before the cause's own message when
+  // that can be printed independently, or a (potentially empty) prefix
-  // constructing a full message. This may contain PII.
+  // which is printed before the cause's own message to construct the
+  // full message. This may contain PII.
   //
   // This is extracted automatically:
   //
   // - for wrappers that have a registered encoder,
   // - otherwise, when the wrapper's Error() has its cause's Error() as suffix.
-  string message_prefix = 2;
+  string message = 2;
 
   // The error details.
   EncodedErrorDetails details = 3 [(gogoproto.nullable) = false];
+
+  // message_type encodes the interpretation of `message`. Prior
+  // versions will not set this field and it will be left as `PREFIX`.
+  // This retains backwards compatibility since the new behavior is
+  // only enabled when this enum is set to `FULL_MESSAGE`.
+  MessageType message_type = 4;
 }
 
 // ErrorTypeMark identifies an error type for the purpose of determining

vendor/github.com/cockroachdb/errors/errutil/as.go

@@ -58,7 +58,16 @@ func As(err error, target interface{}) bool {
 		if x, ok := c.(interface{ As(interface{}) bool }); ok && x.As(target) {
 			return true
 		}
+
+		// If at any point in the single cause chain including the top,
+		// we encounter a multi-cause chain, recursively explore it.
+		for _, cause := range errbase.UnwrapMulti(c) {
+			if As(cause, target) {
+				return true
+			}
+		}
 	}
+
 	return false
 }
 

vendor/github.com/cockroachdb/errors/errutil/doc.go

@@ -14,6 +14,6 @@
 
 // Package errutil combines primitives from the library, the Go errors
 // package and github.com/pkg/errors. It aims to serve as drop-in
-// replacement to gitub.com/pkg/errors and replace the legacy
+// replacement to github.com/pkg/errors and replace the legacy
 // implementation of `pgerrors`.
 package errutil

vendor/github.com/cockroachdb/errors/errutil/utilities.go

@@ -15,6 +15,7 @@
 package errutil
 
 import (
+	"github.com/cockroachdb/errors/join"
 	"github.com/cockroachdb/errors/secondary"
 	"github.com/cockroachdb/errors/withstack"
 	"github.com/cockroachdb/redact"
@@ -158,3 +159,10 @@ func WrapWithDepthf(depth int, err error, format string, args ...interface{}) er
 	err = withstack.WithStackDepth(err, depth+1)
 	return err
 }
+
+// JoinWithDepth constructs a Join error with the provided list of
+// errors as arguments, and wraps it in a `WithStackDepth` to capture a
+// stacktrace alongside.
+func JoinWithDepth(depth int, errs ...error) error {
+	return withstack.WithStackDepth(join.Join(errs...), depth+1)
+}

vendor/github.com/cockroachdb/errors/errutil_api.go

@@ -192,3 +192,19 @@ func HandleAsAssertionFailureDepth(depth int, origErr error) error {
 // - it also supports recursing through causes with Cause().
 // - if it detects an API use error, its panic object is a valid error.
 func As(err error, target interface{}) bool { return errutil.As(err, target) }
+
+// Join returns an error that wraps the given errors.
+// Any nil error values are discarded.
+// Join returns nil if errs contains no non-nil values.
+// The error formats as the concatenation of the strings obtained
+// by calling the Error method of each element of errs, with a newline
+// between each string. A stack trace is also retained.
+func Join(errs ...error) error {
+	return errutil.JoinWithDepth(1, errs...)
+}
+
+// JoinWithDepth is like Join but the depth at which the call stack is
+// captured can be specified.
+func JoinWithDepth(depth int, errs ...error) error {
+	return errutil.JoinWithDepth(depth+1, errs...)
+}

vendor/github.com/cockroachdb/errors/hintdetail/hintdetail.go

@@ -23,6 +23,7 @@ import (
 
 // WithHint decorates an error with a textual hint.
 // The hint may contain PII and thus will not reportable.
+// The suggested use case for hint is to relay information to end users.
 //
 // Hint is shown:
 // - when formatting with `%+v`.
@@ -97,10 +98,12 @@ type ErrorHinter interface {
 
 // WithDetail decorates an error with a textual detail.
 // The detail may contain PII and thus will not reportable.
+// The suggested use case for detail is to augment errors with information
+// useful for debugging.
 //
 // Detail is shown:
 // - when formatting with `%+v`.
-// - with `GetAllHints()` / `FlattenHints()` below.
+// - with `GetAllDetails()` / `FlattenDetails()` below.
 //
 // Note: the detail does not appear in the main error message returned
 // with Error(). Use GetAllDetails() or FlattenDetails() to retrieve

vendor/github.com/cockroachdb/errors/hintdetail_api.go

@@ -27,6 +27,7 @@ type ErrorDetailer = hintdetail.ErrorDetailer
 
 // WithHint decorates an error with a textual hint.
 // The hint may contain PII and thus will not reportable.
+// The suggested use case for hint is to relay information to end users.
 //
 // Hint is shown:
 // - when formatting with `%+v`.
@@ -44,10 +45,12 @@ func WithHintf(err error, format string, args ...interface{}) error {
 
 // WithDetail decorates an error with a textual detail.
 // The detail may contain PII and thus will not reportable.
+// The suggested use case for detail is to augment errors with information
+// useful for debugging.
 //
 // Detail is shown:
 // - when formatting with `%+v`.
-// - with `GetAllHints()` / `FlattenHints()` below.
+// - with `GetAllDetails()` / `FlattenDetails()` below.
 //
 // Note: the detail does not appear in the main error message returned
 // with Error(). Use GetAllDetails() or FlattenDetails() to retrieve

vendor/github.com/cockroachdb/errors/join/join.go

@@ -0,0 +1,105 @@
+// Copyright 2019 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package join
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cockroachdb/errors/errbase"
+	"github.com/cockroachdb/redact"
+	"github.com/gogo/protobuf/proto"
+)
+
+// Join returns an error that wraps the given errors.
+// Any nil error values are discarded.
+// Join returns nil if errs contains no non-nil values.
+// The error formats as the concatenation of the strings obtained
+// by calling the Error method of each element of errs, with a newline
+// between each string.
+func Join(errs ...error) error {
+	n := 0
+	for _, err := range errs {
+		if err != nil {
+			n++
+		}
+	}
+	if n == 0 {
+		return nil
+	}
+	e := &joinError{
+		errs: make([]error, 0, n),
+	}
+	for _, err := range errs {
+		if err != nil {
+			e.errs = append(e.errs, err)
+		}
+	}
+	return e
+}
+
+type joinError struct {
+	errs []error
+}
+
+var _ error = (*joinError)(nil)
+var _ fmt.Formatter = (*joinError)(nil)
+var _ errbase.SafeFormatter = (*joinError)(nil)
+
+func (e *joinError) Error() string {
+	return redact.Sprint(e).StripMarkers()
+}
+
+func (e *joinError) Unwrap() []error {
+	return e.errs
+}
+
+func (e *joinError) SafeFormatError(p errbase.Printer) error {
+	for i, err := range e.errs {
+		if i > 0 {
+			p.Print("\n")
+		}
+		p.Print(err)
+	}
+	return nil
+}
+
+func (e *joinError) Format(s fmt.State, verb rune) {
+	errbase.FormatError(e, s, verb)
+}
+
+func init() {
+	errbase.RegisterMultiCauseEncoder(
+		errbase.GetTypeKey(&joinError{}),
+		func(
+			ctx context.Context,
+			err error,
+		) (msg string, safeDetails []string, payload proto.Message) {
+			return "", nil, nil
+		},
+	)
+	errbase.RegisterMultiCauseDecoder(
+		errbase.GetTypeKey(&joinError{}),
+		func(
+			ctx context.Context,
+			causes []error,
+			msgPrefix string,
+			safeDetails []string,
+			payload proto.Message,
+		) error {
+			return Join(causes...)
+		},
+	)
+}

vendor/github.com/cockroachdb/errors/markers/markers.go

@@ -59,6 +59,13 @@ func Is(err, reference error) bool {
 		if tryDelegateToIsMethod(c, reference) {
 			return true
 		}
+
+		// Recursively try multi-error causes, if applicable.
+		for _, me := range errbase.UnwrapMulti(c) {
+			if Is(me, reference) {
+				return true
+			}
+		}
 	}
 
 	if err == nil {
@@ -68,8 +75,8 @@ func Is(err, reference error) bool {
 		return false
 	}
 
-	// Not directly equal. Try harder, using error marks. We don't this
+	// Not directly equal. Try harder, using error marks. We don't do
-	// during the loop above as it may be more expensive.
+	// this during the loop above as it may be more expensive.
 	//
 	// Note: there is a more effective recursive algorithm that ensures
 	// that any pair of string only gets compared once. Should the
@@ -172,6 +179,13 @@ func IsAny(err error, references ...error) bool {
 				return true
 			}
 		}
+
+		// Recursively try multi-error causes, if applicable.
+		for _, me := range errbase.UnwrapMulti(c) {
+			if IsAny(me, references...) {
+				return true
+			}
+		}
 	}
 
 	// Try harder with marks.

vendor/github.com/cockroachdb/errors/report/report.go

@@ -34,36 +34,39 @@ import (
 //
 // A Sentry report is displayed visually in the Sentry UI as follows:
 //
-////////////////
+// //////////////
 // Title: (1) some prefix in bold (2) one line for a stack trace
 // (3) a single-line subtitle
 //
-// (4) the tags, as a tag soup (concatenated in a single paragrah,
+// (4) the tags, as a tag soup (concatenated in a single paragraph,
 // unsorted)
 //
 // (5) a "message"
 //
 // (6) zero or more "exceptions", each composed of:
-//    (7) a bold title
+//
-//    (8) some freeform text
+//	(7) a bold title
-//    (9) a stack trace
+//	(8) some freeform text
+//	(9) a stack trace
 //
 // (10) metadata fields: environment, arch, etc
 //
 // (11) "Additional data" fields
 //
 // (12) SDK version
-/////////////////
+// ///////////////
 //
 // These visual items map to the Sentry Event object as follows:
 //
 // (1) the Type field of the 1st Exception object, if any
-//     otherwise the Message field
+//
+//	otherwise the Message field
+//
 // (2) the topmost entry from the Stacktrace field of the 1st Exception object, if any
 // (3) the Value field of the 1st Exception object, if any, unwrapped as a single line
 // (4) the Tags field
 // (5) the Message field
-// (7) the Type field (same as (1) for 1st execption)
+// (7) the Type field (same as (1) for 1st exception)
 // (8) the Value field (same as (3) for 1st exception)
 // (9) the Stacktrace field (input to (2) on 1st exception)
 // (10) the other fields on the Event object
@@ -78,7 +81,9 @@ import (
 // (3)/(8): first line of verbose error printout
 // (4): not populated in this function, caller is to manage this
 // (5): detailed structure of the entire error object, with references to
-//      additional "exception" objects
+//
+//	additional "exception" objects
+//
 // (9): generated from innermost stack trace
 // (6): every exception object after the 1st reports additional stack trace contexts
 // (11): the detailed error types and their error mark.
@@ -92,7 +97,6 @@ import (
 // is included in the Sentry report. This does not affect error types
 // provided by the library, but could impact error types defined by
 // 3rd parties. This limitation may be lifted in a later version.
-//
 func BuildSentryReport(err error) (event *sentry.Event, extraDetails map[string]interface{}) {
 	if err == nil {
 		// No error: do nothing.
@@ -102,13 +106,13 @@ func BuildSentryReport(err error) (event *sentry.Event, extraDetails map[string]
 	// First step is to collect the details.
 	var stacks []*withstack.ReportableStackTrace
 	var details []errbase.SafeDetailPayload
-	for c := err; c != nil; c = errbase.UnwrapOnce(c) {
+	visitAllMulti(err, func(c error) {
 		st := withstack.GetReportableStackTrace(c)
 		stacks = append(stacks, st)
 
 		sd := errbase.GetSafeDetails(c)
 		details = append(details, sd)
-	}
+	})
 	module := string(domains.GetDomain(err))
 
 	// firstDetailLine is the first detail string encountered.
@@ -292,7 +296,7 @@ func BuildSentryReport(err error) (event *sentry.Event, extraDetails map[string]
 	event.Message = longMsgBuf.String()
 	event.Exception = exceptions
 
-	// If there is no exception payload, synthetize one.
+	// If there is no exception payload, synthesize one.
 	if len(event.Exception) == 0 {
 		// We know we don't have a stack trace to extract line/function
 		// info from (if we had, we'd have an Exception payload at that
@@ -397,3 +401,13 @@ func reverseExceptionOrder(ex []sentry.Exception) {
 		ex[i], ex[len(ex)-i-1] = ex[len(ex)-i-1], ex[i]
 	}
 }
+
+func visitAllMulti(err error, f func(error)) {
+	f(err)
+	if e := errbase.UnwrapOnce(err); e != nil {
+		visitAllMulti(e, f)
+	}
+	for _, e := range errbase.UnwrapMulti(err) {
+		visitAllMulti(e, f)
+	}
+}

vendor/github.com/cockroachdb/errors/report_api.go

@@ -32,7 +32,7 @@ import (
 // Title: (1) some prefix in bold (2) one line for a stack trace
 // (3) a single-line subtitle
 //
-// (4) the tags, as a tag soup (concatenated in a single paragrah,
+// (4) the tags, as a tag soup (concatenated in a single paragraph,
 // unsorted)
 //
 // (5) a "message"
@@ -57,7 +57,7 @@ import (
 // (3) the Value field of the 1st Exception object, if any, unwrapped as a single line
 // (4) the Tags field
 // (5) the Message field
-// (7) the Type field (same as (1) for 1st execption)
+// (7) the Type field (same as (1) for 1st exception)
 // (8) the Value field (same as (3) for 1st exception)
 // (9) the Stacktrace field (input to (2) on 1st exception)
 // (10) the other fields on the Event object

vendor/github.com/cockroachdb/fifo/.gitignore

@@ -0,0 +1,25 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+.idea/
+

vendor/github.com/cockroachdb/fifo/LICENSE

@@ -178,7 +178,7 @@
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
+      boilerplate notice, with the fields enclosed by brackets "[]"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

vendor/github.com/cockroachdb/fifo/invariants_off.go

@@ -0,0 +1,21 @@
+// Copyright 2024 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+//go:build !fifo_invariants
+
+package fifo
+
+// invariants is false if we were not built with the "fifo_invariants" build
+// tag.
+const invariants = false

vendor/github.com/cockroachdb/fifo/invariants_on.go

@@ -0,0 +1,20 @@
+// Copyright 2024 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+//go:build fifo_invariants
+
+package fifo
+
+// invariants is true if we were built with the "fifo_invariants" build tag.
+const invariants = true

vendor/github.com/cockroachdb/fifo/queue.go

@@ -0,0 +1,162 @@
+// Copyright 2024 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package fifo
+
+import "sync"
+
+// Queue implements an allocation efficient FIFO queue. It is not safe for
+// concurrent access.
+//
+// Note that the queue provides pointer access to the internal storage (via
+// PeekFront and PushBack) so it must be used with care. These pointers must not
+// be used once the respective element is popped out of the queue.
+//
+// -- Implementation --
+//
+// The queue is implemented as a linked list of nodes, where each node is a
+// small ring buffer. The nodes are allocated using a sync.Pool (a single pool
+// is created for any given type and is used for all queues of that type).
+type Queue[T any] struct {
+	len        int
+	head, tail *queueNode[T]
+
+	pool *QueueBackingPool[T]
+}
+
+// MakeQueue constructs a new Queue.
+//
+// The pool should be a singleton object initialized with MakeQueueBackingPool.
+// A single pool can and should be used by all queues of that type.
+func MakeQueue[T any](pool *QueueBackingPool[T]) Queue[T] {
+	return Queue[T]{
+		pool: pool,
+	}
+}
+
+// Len returns the current length of the queue.
+func (q *Queue[T]) Len() int {
+	return q.len
+}
+
+// PushBack adds t to the end of the queue.
+//
+// The returned pointer can be used to modify the element while it is in the
+// queue; it is valid until the element is removed from the queue.
+func (q *Queue[T]) PushBack(t T) *T {
+	if q.head == nil {
+		q.head = q.pool.get()
+		q.tail = q.head
+	} else if q.tail.IsFull() {
+		newTail := q.pool.get()
+		q.tail.next = newTail
+		q.tail = newTail
+	}
+	q.len++
+	return q.tail.PushBack(t)
+}
+
+// PeekFront returns the current head of the queue, or nil if the queue is
+// empty.
+//
+// The result is only valid until the next call to PopFront.
+func (q *Queue[T]) PeekFront() *T {
+	if q.len == 0 {
+		return nil
+	}
+	return q.head.PeekFront()
+}
+
+// PopFront removes the current head of the queue.
+//
+// It is illegal to call PopFront on an empty queue.
+func (q *Queue[T]) PopFront() {
+	q.head.PopFront()
+	// If this is the only node, we don't want to release it; otherwise we would
+	// allocate/free a node every time we transition between the queue being empty
+	// and non-empty.
+	if q.head.len == 0 && q.head.next != nil {
+		oldHead := q.head
+		q.head = oldHead.next
+		q.pool.put(oldHead)
+	}
+	q.len--
+}
+
+// QueueBackingPool is a sync.Pool that used to allocate internal nodes
+// for Queue[T].
+type QueueBackingPool[T any] struct {
+	pool sync.Pool
+}
+
+// MakeQueueBackingPool makes a queue backing pool. It is intented to be used to
+// initialize a singleton (global) variable. A single pool can and should be
+// used by all queues of that type.
+func MakeQueueBackingPool[T any]() QueueBackingPool[T] {
+	return QueueBackingPool[T]{
+		pool: sync.Pool{
+			New: func() interface{} { return &queueNode[T]{} },
+		},
+	}
+}
+
+func (qp *QueueBackingPool[T]) get() *queueNode[T] {
+	return qp.pool.Get().(*queueNode[T])
+}
+
+func (qp *QueueBackingPool[T]) put(n *queueNode[T]) {
+	*n = queueNode[T]{}
+	qp.pool.Put(n)
+}
+
+// We batch the allocation of this many queue objects. The value was chosen
+// without experimentation - it provides a reasonable amount of amortization
+// without a very large increase in memory overhead if T is large.
+const queueNodeSize = 8
+
+type queueNode[T any] struct {
+	buf       [queueNodeSize]T
+	head, len int32
+	next      *queueNode[T]
+}
+
+func (qn *queueNode[T]) IsFull() bool {
+	return qn.len == queueNodeSize
+}
+
+func (qn *queueNode[T]) PushBack(t T) *T {
+	if invariants && qn.len >= queueNodeSize {
+		panic("cannot push back into a full node")
+	}
+	i := (qn.head + qn.len) % queueNodeSize
+	qn.buf[i] = t
+	qn.len++
+	return &qn.buf[i]
+}
+
+func (qn *queueNode[T]) PeekFront() *T {
+	return &qn.buf[qn.head]
+}
+
+func (qn *queueNode[T]) PopFront() T {
+	if invariants && qn.len == 0 {
+		panic("cannot pop from empty queue")
+	}
+	t := qn.buf[qn.head]
+	var zero T
+	qn.buf[qn.head] = zero
+	qn.head = (qn.head + 1) % queueNodeSize
+	qn.len--
+	return t
+}

vendor/github.com/cockroachdb/fifo/semaphore.go

@@ -0,0 +1,239 @@
+// Copyright 2024 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package fifo
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+)
+
+// Semaphore implements a weighted, dynamically reconfigurable semaphore which
+// respects context cancellation.
+//
+// The semaphore implements a FIFO policy, where Acquire requests are satisfied
+// in order. This policy provides fairness and prevents starvation but is
+// susceptible to head-of-line blocking, where a large request that can't be
+// satisfied blocks many other small requests that could be.
+type Semaphore struct {
+	mu struct {
+		sync.Mutex
+
+		capacity int64
+		// outstanding can exceed capacity if the capacity is dynamically decreased.
+		outstanding int64
+
+		waiters Queue[semaWaiter]
+
+		// numCanceled is the number of waiters in the waiters queue which have been
+		// canceled. It is used to determine the current number of active waiters in
+		// the queue which is waiters.Len() minus this value.
+		numCanceled int
+
+		// numHadToWait accumulates the total number of Acquire requests which had
+		// to wait because the semaphore was exhausted.
+		numHadToWait int64
+	}
+}
+
+// NewSemaphore creates a new semaphore with the given capacity.
+func NewSemaphore(capacity int64) *Semaphore {
+	if capacity <= 0 {
+		panic("invalid capacity")
+	}
+	s := &Semaphore{}
+	s.mu.capacity = capacity
+	s.mu.waiters = MakeQueue[semaWaiter](&semaQueuePool)
+	return s
+}
+
+var semaQueuePool = MakeQueueBackingPool[semaWaiter]()
+
+var ErrRequestExceedsCapacity = errors.New("request exceeds semaphore capacity")
+
+// TryAcquire attempts to acquire n units from the semaphore without waiting. On
+// success, returns true and the caller must later Release the units.
+func (s *Semaphore) TryAcquire(n int64) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if s.numWaitersLocked() == 0 && s.mu.outstanding+n <= s.mu.capacity {
+		s.mu.outstanding += n
+		return true
+	}
+
+	return false
+}
+
+// Acquire n units from the semaphore, waiting if necessary.
+//
+// If the context is canceled while we are waiting, returns the context error.
+// If n exceeds the current capacity, returns ErrRequestExceedsCapacity.
+// On success, the caller must later Release the units.
+func (s *Semaphore) Acquire(ctx context.Context, n int64) error {
+	s.mu.Lock()
+
+	// Fast path.
+	if s.numWaitersLocked() == 0 && s.mu.outstanding+n <= s.mu.capacity {
+		s.mu.outstanding += n
+		s.mu.Unlock()
+		return nil
+	}
+
+	if n > s.mu.capacity {
+		s.mu.Unlock()
+		return ErrRequestExceedsCapacity
+	}
+
+	c := chanSyncPool.Get().(chan error)
+	defer chanSyncPool.Put(c)
+	w := s.mu.waiters.PushBack(semaWaiter{n: n, c: c})
+	s.mu.numHadToWait++
+	s.mu.Unlock()
+
+	select {
+	case <-ctx.Done():
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		// We need to check if we raced with a channel notify (which happens under
+		// the lock).
+		select {
+		case err := <-c:
+			// We actually fulfilled or failed the request.
+			return err
+		default:
+		}
+		// Mark the request as canceled.
+		w.c = nil
+		s.mu.numCanceled++
+		// If we are the head of the queue, we may be able to fulfill other waiters.
+		s.processWaitersLocked()
+		return ctx.Err()
+
+	case err := <-c:
+		return err
+	}
+}
+
+// Release n units back. These must be units that were acquired by a previous
+// Acquire call. It is legal to split up or coalesce units when releasing.
+func (s *Semaphore) Release(n int64) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.mu.outstanding -= n
+	if s.mu.outstanding < 0 {
+		panic("releasing more than was acquired")
+	}
+	s.processWaitersLocked()
+}
+
+// UpdateCapacity changes the capacity of the semaphore. If the new capacity is
+// smaller, the already outstanding acquisitions might exceed the new capacity
+// until they are released.
+//
+// If there are Acquire calls that are waiting which are requesting more than
+// the new capacity, they will error out with ErrRequestExceedsCapacity.
+func (s *Semaphore) UpdateCapacity(capacity int64) {
+	if capacity <= 0 {
+		panic("invalid capacity")
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.mu.capacity = capacity
+	s.processWaitersLocked()
+}
+
+// Stats returns the current state of the semaphore.
+func (s *Semaphore) Stats() SemaphoreStats {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return SemaphoreStats{
+		Capacity:     s.mu.capacity,
+		Outstanding:  s.mu.outstanding,
+		NumHadToWait: s.mu.numHadToWait,
+	}
+}
+
+// SemaphoreStats contains information about the current state of the semaphore.
+type SemaphoreStats struct {
+	// Capacity is the current capacity of the semaphore.
+	Capacity int64
+	// Outstanding is the number of units that have been acquired. Note that this
+	// can exceed Capacity if the capacity was recently decreased.
+	Outstanding int64
+	// NumHadToWait is the total number of Acquire calls (since the semaphore was
+	// created) that had to wait because the semaphore was exhausted. Useful for
+	// cumulative metrics.
+	NumHadToWait int64
+
+	// TODO(radu): consider keeping track of the total amount of time the
+	// semaphore was exhausted (i.e. there were waiters queued).
+}
+
+func (ss SemaphoreStats) String() string {
+	return fmt.Sprintf("capacity: %d, outstanding: %d, num-had-to-wait: %d",
+		ss.Capacity, ss.Outstanding, ss.NumHadToWait)
+}
+
+type semaWaiter struct {
+	// n is the amount that the waiter is trying to acquire.
+	n int64
+	// c is the channel on which Acquire is blocked. If the request is canceled,
+	// it is set to nil.
+	c chan error
+}
+
+// numWaitersLocked returns how many requests (that have not been canceled) are
+// waiting in the queue.
+func (s *Semaphore) numWaitersLocked() int {
+	return s.mu.waiters.Len() - s.mu.numCanceled
+}
+
+// processWaitersLocked processes and notifies as many waiters from the head of
+// the queue as possible.
+func (s *Semaphore) processWaitersLocked() {
+	for ; s.mu.waiters.Len() > 0; s.mu.waiters.PopFront() {
+		switch w := s.mu.waiters.PeekFront(); {
+		case w.c == nil:
+			// Request was canceled, we can just clean it up.
+			s.mu.numCanceled--
+			if invariants && s.mu.numCanceled < 0 {
+				panic("negative numCanceled")
+			}
+
+		case s.mu.outstanding+w.n <= s.mu.capacity:
+			// Request can be fulfilled.
+			s.mu.outstanding += w.n
+			w.c <- nil
+
+		case w.n > s.mu.capacity:
+			// Request must be failed. This can happen if the capacity was decreased
+			// while the element was queued.
+			w.c <- ErrRequestExceedsCapacity
+
+		default:
+			// Head of the queue needs to wait some more.
+			return
+		}
+	}
+}
+
+// chanSyncPool is used to pool allocations of the channels used to notify
+// goroutines waiting in Acquire.
+var chanSyncPool = sync.Pool{
+	New: func() interface{} { return make(chan error, 1) },
+}

vendor/github.com/cockroachdb/pebble/.gitignore

@@ -1,6 +1,9 @@
+# Github action artifacts.
+artifacts
 # Profiling artifacts.
 cpu.*.prof
 heap.prof
 mutex.prof
+coverprofile.out
 # Testing artifacts
 meta.*.test

vendor/github.com/cockroachdb/pebble/Makefile

@@ -4,7 +4,8 @@ GOFLAGS :=
 STRESSFLAGS :=
 TAGS := invariants
 TESTS := .
-LATEST_RELEASE := $(shell git fetch origin && git branch -r --list '*/crl-release-*' | grep -o 'crl-release-.*$$' | sort | tail -1)
+PREV_RELEASE := crl-release-23.1
+COVER_PROFILE := coverprofile.out
 
 .PHONY: all
 all:
@@ -15,6 +16,7 @@ all:
 	@echo "  make stressrace"
 	@echo "  make stressmeta"
 	@echo "  make crossversion-meta"
+	@echo "  make testcoverage"
 	@echo "  make mod-update"
 	@echo "  make generate"
 	@echo "  make generate-test-data"
@@ -25,6 +27,10 @@ override testflags :=
 test:
 	${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG}
 
+.PHONY: testcoverage
+testcoverage:
+	${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG} -coverprofile ${COVER_PROFILE}
+
 .PHONY: testrace
 testrace: testflags += -race -timeout 20m
 testrace: test
@@ -57,11 +63,11 @@ stressmeta: stress
 
 .PHONY: crossversion-meta
 crossversion-meta:
-	git checkout ${LATEST_RELEASE}; \
+	git checkout ${PREV_RELEASE}; \
-		${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${LATEST_RELEASE}.test'; \
+		${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${PREV_RELEASE}.test'; \
 		git checkout -; \
 		${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/head.test'; \
-		${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${LATEST_RELEASE},${LATEST_RELEASE},${LATEST_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
+		${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${PREV_RELEASE},${PREV_RELEASE},${PREV_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
 
 .PHONY: stress-crossversion
 stress-crossversion:
@@ -80,6 +86,7 @@ generate-test-data:
 	${GO} run -tags make_incorrect_manifests ./tool/make_incorrect_manifests.go
 	${GO} run -tags make_test_find_db ./tool/make_test_find_db.go
 	${GO} run -tags make_test_sstables ./tool/make_test_sstables.go
+	${GO} run -tags make_test_remotecat ./tool/make_test_remotecat.go
 
 mod-update:
 	${GO} get -u

vendor/github.com/cockroachdb/pebble/README.md

@@ -1,4 +1,4 @@
-# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble)
+# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble) <sup><sub><sub>[Coverage](https://storage.googleapis.com/crl-codecover-public/pebble/index.html)</sub></sub></sup>
 
 #### [Nightly benchmarks](https://cockroachdb.github.io/pebble/)
 
@@ -8,9 +8,8 @@ RocksDB file formats and a few extensions such as range deletion
 tombstones, table-level bloom filters, and updates to the MANIFEST
 format.
 
-Pebble intentionally does not aspire to include every feature in
+Pebble intentionally does not aspire to include every feature in RocksDB and
-RocksDB and is specifically targetting the use case and feature set
+specifically targets the use case and feature set needed by CockroachDB:
-needed by CockroachDB:
 
 * Block-based tables
 * Checkpoints
@@ -68,6 +67,17 @@ Pebble offers several improvements over RocksDB:
 * Faster commit pipeline that achieves better concurrency.
 * Seamless merged iteration of indexed batches. The mutations in the
   batch conceptually occupy another memtable level.
+* L0 sublevels and flush splitting for concurrent compactions out of L0 and
+  reduced read-amplification during heavy write load.
+* Faster LSM edits in LSMs with large numbers of sstables through use of a
+  copy-on-write B-tree to hold file metadata.
+* Delete-only compactions that drop whole sstables that fall within the bounds
+  of a range deletion.
+* Block-property collectors and filters that enable iterators to skip tables,
+  index blocks and data blocks that are irrelevant, according to user-defined
+  properties over key-value pairs.
+* Range keys API, allowing KV pairs defined over a range of keyspace with
+  user-defined semantics and interleaved during iteration.
 * Smaller, more approachable code base.
 
 See the [Pebble vs RocksDB: Implementation
@@ -100,10 +110,68 @@ known incompatibilities.
   format.
 * Plain table format. Pebble does not support the plain table sstable
   format.
-* SSTable format version 3 and 4. Pebble does not currently support
+* SSTable format version 3 and 4. Pebble does not support version 3
-  version 3 and version 4 format sstables. The sstable format version
+  and version 4 format sstables. The sstable format version is
-  is controlled by the `BlockBasedTableOptions::format_version`
+  controlled by the `BlockBasedTableOptions::format_version` option.
-  option. See [#97](https://github.com/cockroachdb/pebble/issues/97).
+  See [#97](https://github.com/cockroachdb/pebble/issues/97).
+
+## Format major versions
+
+Over time Pebble has introduced new physical file formats.  Backwards
+incompatible changes are made through the introduction of 'format major
+versions'. By default, when Pebble opens a database, it defaults to
+`FormatMostCompatible`. This version is bi-directionally compatible with RocksDB
+6.2.1 (with the caveats described above).
+
+To opt into new formats, a user may set `FormatMajorVersion` on the
+[`Options`](https://pkg.go.dev/github.com/cockroachdb/pebble#Options)
+supplied to
+[`Open`](https://pkg.go.dev/github.com/cockroachdb/pebble#Open), or
+upgrade the format major version at runtime using
+[`DB.RatchetFormatMajorVersion`](https://pkg.go.dev/github.com/cockroachdb/pebble#DB.RatchetFormatMajorVersion).
+Format major version upgrades are permanent; There is no option to
+return to an earlier format.
+
+The table below outlines the history of format major versions:
+
+| Name                               | Value | Migration  |
+|------------------------------------|-------|------------|
+| FormatMostCompatible               |   1   | No         |
+| FormatVersioned                    |   3   | No         |
+| FormatSetWithDelete                |   4   | No         |
+| FormatBlockPropertyCollector       |   5   | No         |
+| FormatSplitUserKeysMarked          |   6   | Background |
+| FormatSplitUserKeysMarkedCompacted |   7   | Blocking   |
+| FormatRangeKeys                    |   8   | No         |
+| FormatMinTableFormatPebblev1       |   9   | No         |
+| FormatPrePebblev1Marked            |  10   | Background |
+| FormatSSTableValueBlocks           |  12   | No         |
+| FormatFlushableIngest              |  13   | No         |
+| FormatPrePebblev1MarkedCompacted   |  14   | Blocking   |
+| FormatDeleteSizedAndObsolete       |  15   | No         |
+| FormatVirtualSSTables              |  16   | No         |
+
+Upgrading to a format major version with 'Background' in the migration
+column may trigger background activity to rewrite physical file
+formats, typically through compactions. Upgrading to a format major
+version with 'Blocking' in the migration column will block until a
+migration is complete. The database may continue to serve reads and
+writes if upgrading a live database through
+`RatchetFormatMajorVersion`, but the method call will not return until
+the migration is complete.
+
+For reference, the table below lists the range of supported Pebble format major
+versions for CockroachDB releases.
+
+| CockroachDB release | Earliest supported                 | Latest supported          |
+|---------------------|------------------------------------|---------------------------|
+| 20.1 through 21.1   | FormatMostCompatible               | FormatMostCompatible      |
+| 21.2                | FormatMostCompatible               | FormatSetWithDelete       |
+| 21.2                | FormatMostCompatible               | FormatSetWithDelete       |
+| 22.1                | FormatMostCompatible               | FormatSplitUserKeysMarked |
+| 22.2                | FormatMostCompatible               | FormatPrePebblev1Marked   |
+| 23.1                | FormatSplitUserKeysMarkedCompacted | FormatFlushableIngest     |
+| 23.2                | FormatSplitUserKeysMarkedCompacted | FormatVirtualSSTables     |
 
 ## Pedigree
 

vendor/github.com/cockroachdb/pebble/batch.go

@@ -41,10 +41,10 @@ const (
 var ErrNotIndexed = errors.New("pebble: batch not indexed")
 
 // ErrInvalidBatch indicates that a batch is invalid or otherwise corrupted.
-var ErrInvalidBatch = errors.New("pebble: invalid batch")
+var ErrInvalidBatch = base.MarkCorruptionError(errors.New("pebble: invalid batch"))
 
 // ErrBatchTooLarge indicates that a batch is invalid or otherwise corrupted.
-var ErrBatchTooLarge = errors.Newf("pebble: batch too large: >= %s", humanize.Uint64(maxBatchSize))
+var ErrBatchTooLarge = base.MarkCorruptionError(errors.Newf("pebble: batch too large: >= %s", humanize.Bytes.Uint64(maxBatchSize)))
 
 // DeferredBatchOp represents a batch operation (eg. set, merge, delete) that is
 // being inserted into the batch. Indexing is not performed on the specified key
@@ -183,6 +183,13 @@ func (d DeferredBatchOp) Finish() error {
 // WAL, and thus stable. New record kinds may be added, but the existing ones
 // will not be modified.
 type Batch struct {
+	batchInternal
+	applied atomic.Bool
+}
+
+// batchInternal contains the set of fields within Batch that are non-atomic and
+// capable of being reset using a *b = batchInternal{} struct copy.
+type batchInternal struct {
 	// Data is the wire format of a batch's log entry:
 	//   - 8 bytes for a sequence number of the first batch element,
 	//     or zeroes if the batch has not yet been applied,
@@ -260,11 +267,6 @@ type Batch struct {
 	// memtable.
 	flushable *flushableBatch
 
-	// ingestedSSTBatch indicates that the batch contains one or more key kinds
-	// of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST
-	// then it will only contain key kinds of IngestSST.
-	ingestedSSTBatch bool
-
 	// minimumFormatMajorVersion indicates the format major version required in
 	// order to commit this batch. If an operation requires a particular format
 	// major version, it ratchets the batch's minimumFormatMajorVersion. When
@@ -292,7 +294,22 @@ type Batch struct {
 	commitStats BatchCommitStats
 
 	commitErr error
-	applied   atomic.Bool
+
+	// Position bools together to reduce the sizeof the struct.
+
+	// ingestedSSTBatch indicates that the batch contains one or more key kinds
+	// of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST
+	// then it will only contain key kinds of IngestSST.
+	ingestedSSTBatch bool
+
+	// committing is set to true when a batch begins to commit. It's used to
+	// ensure the batch is not mutated concurrently. It is not an atomic
+	// deliberately, so as to avoid the overhead on batch mutations. This is
+	// okay, because under correct usage this field will never be accessed
+	// concurrently. It's only under incorrect usage the memory accesses of this
+	// variable may violate memory safety. Since we don't use atomics here,
+	// false negatives are possible.
+	committing bool
 }
 
 // BatchCommitStats exposes stats related to committing a batch.
@@ -319,7 +336,11 @@ type BatchCommitStats struct {
 	// commitPipeline.Commit.
 	SemaphoreWaitDuration time.Duration
 	// WALQueueWaitDuration is the wait time for allocating memory blocks in the
-	// LogWriter (due to the LogWriter not writing fast enough).
+	// LogWriter (due to the LogWriter not writing fast enough). At the moment
+	// this is duration is always zero because a single WAL will allow
+	// allocating memory blocks up to the entire memtable size. In the future,
+	// we may pipeline WALs and bound the WAL queued blocks separately, so this
+	// field is preserved for that possibility.
 	WALQueueWaitDuration time.Duration
 	// MemTableWriteStallDuration is the wait caused by a write stall due to too
 	// many memtables (due to not flushing fast enough).
@@ -363,6 +384,14 @@ func newBatch(db *DB) *Batch {
 	return b
 }
 
+func newBatchWithSize(db *DB, size int) *Batch {
+	b := newBatch(db)
+	if cap(b.data) < size {
+		b.data = rawalloc.New(0, size)
+	}
+	return b
+}
+
 func newIndexedBatch(db *DB, comparer *Comparer) *Batch {
 	i := indexedBatchPool.Get().(*indexedBatch)
 	i.batch.cmp = comparer.Compare
@@ -374,6 +403,14 @@ func newIndexedBatch(db *DB, comparer *Comparer) *Batch {
 	return &i.batch
 }
 
+func newIndexedBatchWithSize(db *DB, comparer *Comparer, size int) *Batch {
+	b := newIndexedBatch(db, comparer)
+	if cap(b.data) < size {
+		b.data = rawalloc.New(0, size)
+	}
+	return b
+}
+
 // nextSeqNum returns the batch "sequence number" that will be given to the next
 // key written to the batch. During iteration keys within an indexed batch are
 // given a sequence number consisting of their offset within the batch combined
@@ -412,18 +449,21 @@ func (b *Batch) release() {
 	}
 }
 
-func (b *Batch) refreshMemTableSize() {
+func (b *Batch) refreshMemTableSize() error {
 	b.memTableSize = 0
 	if len(b.data) < batchHeaderLen {
-		return
+		return nil
 	}
 
 	b.countRangeDels = 0
 	b.countRangeKeys = 0
 	b.minimumFormatMajorVersion = 0
 	for r := b.Reader(); ; {
-		kind, key, value, ok := r.Next()
+		kind, key, value, ok, err := r.Next()
 		if !ok {
+			if err != nil {
+				return err
+			}
 			break
 		}
 		switch kind {
@@ -432,8 +472,8 @@ func (b *Batch) refreshMemTableSize() {
 		case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
 			b.countRangeKeys++
 		case InternalKeyKindDeleteSized:
-			if b.minimumFormatMajorVersion < ExperimentalFormatDeleteSizedAndObsolete {
+			if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete {
-				b.minimumFormatMajorVersion = ExperimentalFormatDeleteSizedAndObsolete
+				b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete
 			}
 		case InternalKeyKindIngestSST:
 			if b.minimumFormatMajorVersion < FormatFlushableIngest {
@@ -447,6 +487,7 @@ func (b *Batch) refreshMemTableSize() {
 	if b.countRangeKeys > 0 && b.minimumFormatMajorVersion < FormatRangeKeys {
 		b.minimumFormatMajorVersion = FormatRangeKeys
 	}
+	return nil
 }
 
 // Apply the operations contained in the batch to the receiver batch.
@@ -460,7 +501,7 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error {
 		return nil
 	}
 	if len(batch.data) < batchHeaderLen {
-		return base.CorruptionErrorf("pebble: invalid batch")
+		return ErrInvalidBatch
 	}
 
 	offset := len(b.data)
@@ -477,8 +518,11 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error {
 		// order to update the index.
 		for iter := BatchReader(b.data[offset:]); len(iter) > 0; {
 			offset := uintptr(unsafe.Pointer(&iter[0])) - uintptr(unsafe.Pointer(&b.data[0]))
-			kind, key, value, ok := iter.Next()
+			kind, key, value, ok, err := iter.Next()
 			if !ok {
+				if err != nil {
+					return err
+				}
 				break
 			}
 			switch kind {
@@ -534,6 +578,9 @@ func (b *Batch) Get(key []byte) ([]byte, io.Closer, error) {
 }
 
 func (b *Batch) prepareDeferredKeyValueRecord(keyLen, valueLen int, kind InternalKeyKind) {
+	if b.committing {
+		panic("pebble: batch already committing")
+	}
 	if len(b.data) == 0 {
 		b.init(keyLen + valueLen + 2*binary.MaxVarintLen64 + batchHeaderLen)
 	}
@@ -583,6 +630,9 @@ func (b *Batch) prepareDeferredKeyValueRecord(keyLen, valueLen int, kind Interna
 }
 
 func (b *Batch) prepareDeferredKeyRecord(keyLen int, kind InternalKeyKind) {
+	if b.committing {
+		panic("pebble: batch already committing")
+	}
 	if len(b.data) == 0 {
 		b.init(keyLen + binary.MaxVarintLen64 + batchHeaderLen)
 	}
@@ -764,8 +814,8 @@ func (b *Batch) DeleteSized(key []byte, deletedValueSize uint32, _ *WriteOptions
 // complete key slice, letting the caller encode into the DeferredBatchOp.Key
 // slice and then call Finish() on the returned object.
 func (b *Batch) DeleteSizedDeferred(keyLen int, deletedValueSize uint32) *DeferredBatchOp {
-	if b.minimumFormatMajorVersion < ExperimentalFormatDeleteSizedAndObsolete {
+	if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete {
-		b.minimumFormatMajorVersion = ExperimentalFormatDeleteSizedAndObsolete
+		b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete
 	}
 
 	// Encode the sum of the key length and the value in the value.
@@ -1053,11 +1103,12 @@ func (b *Batch) SetRepr(data []byte) error {
 	}
 	b.data = data
 	b.count = uint64(binary.LittleEndian.Uint32(b.countData()))
+	var err error
 	if b.db != nil {
 		// Only track memTableSize for batches that will be committed to the DB.
-		b.refreshMemTableSize()
+		err = b.refreshMemTableSize()
 	}
-	return nil
+	return err
 }
 
 // NewIter returns an iterator that is unpositioned (Iterator.Valid() will
@@ -1067,8 +1118,8 @@ func (b *Batch) SetRepr(data []byte) error {
 // The returned Iterator observes all of the Batch's existing mutations, but no
 // later mutations. Its view can be refreshed via RefreshBatchSnapshot or
 // SetOptions().
-func (b *Batch) NewIter(o *IterOptions) *Iterator {
+func (b *Batch) NewIter(o *IterOptions) (*Iterator, error) {
-	return b.NewIterWithContext(context.Background(), o)
+	return b.NewIterWithContext(context.Background(), o), nil
 }
 
 // NewIterWithContext is like NewIter, and additionally accepts a context for
@@ -1077,7 +1128,7 @@ func (b *Batch) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterato
 	if b.index == nil {
 		return &Iterator{err: ErrNotIndexed}
 	}
-	return b.db.newIter(ctx, b, nil /* snapshot */, o)
+	return b.db.newIter(ctx, b, snapshotIterOpts{}, o)
 }
 
 // newInternalIter creates a new internalIterator that iterates over the
@@ -1290,39 +1341,40 @@ func (b *Batch) Indexed() bool {
 	return b.index != nil
 }
 
-func (b *Batch) init(cap int) {
+// init ensures that the batch data slice is initialized to meet the
+// minimum required size and allocates space for the batch header.
+func (b *Batch) init(size int) {
 	n := batchInitialSize
-	for n < cap {
+	for n < size {
 		n *= 2
 	}
-	b.data = rawalloc.New(batchHeaderLen, n)
+	if cap(b.data) < n {
-	b.setCount(0)
+		b.data = rawalloc.New(batchHeaderLen, n)
-	b.setSeqNum(0)
+	}
 	b.data = b.data[:batchHeaderLen]
+	// Zero the sequence number in the header.
+	for i := 0; i < len(b.data); i++ {
+		b.data[i] = 0
+	}
 }
 
 // Reset resets the batch for reuse. The underlying byte slice (that is
-// returned by Repr()) is not modified. It is only necessary to call this
+// returned by Repr()) may not be modified. It is only necessary to call this
 // method if a batch is explicitly being reused. Close automatically takes are
 // of releasing resources when appropriate for batches that are internally
 // being reused.
 func (b *Batch) Reset() {
-	b.count = 0
+	// Zero out the struct, retaining only the fields necessary for manual
-	b.countRangeDels = 0
+	// reuse.
-	b.countRangeKeys = 0
+	b.batchInternal = batchInternal{
-	b.memTableSize = 0
+		data:           b.data,
-	b.deferredOp = DeferredBatchOp{}
+		cmp:            b.cmp,
-	b.tombstones = nil
+		formatKey:      b.formatKey,
-	b.tombstonesSeqNum = 0
+		abbreviatedKey: b.abbreviatedKey,
-	b.rangeKeys = nil
+		index:          b.index,
-	b.rangeKeysSeqNum = 0
+		db:             b.db,
-	b.flushable = nil
+	}
-	b.commit = sync.WaitGroup{}
-	b.fsyncWait = sync.WaitGroup{}
-	b.commitStats = BatchCommitStats{}
-	b.commitErr = nil
 	b.applied.Store(false)
-	b.minimumFormatMajorVersion = 0
 	if b.data != nil {
 		if cap(b.data) > batchMaxRetainedSize {
 			// If the capacity of the buffer is larger than our maximum
@@ -1333,13 +1385,14 @@ func (b *Batch) Reset() {
 		} else {
 			// Otherwise, reset the buffer for re-use.
 			b.data = b.data[:batchHeaderLen]
-			b.setSeqNum(0)
+			// Zero the sequence number in the header.
+			for i := 0; i < len(b.data); i++ {
+				b.data[i] = 0
+			}
 		}
 	}
 	if b.index != nil {
 		b.index.Init(&b.data, b.cmp, b.abbreviatedKey)
-		b.rangeDelIndex = nil
-		b.rangeKeyIndex = nil
 	}
 }
 
@@ -1411,6 +1464,11 @@ func (b *Batch) Reader() BatchReader {
 }
 
 func batchDecodeStr(data []byte) (odata []byte, s []byte, ok bool) {
+	// TODO(jackson): This will index out of bounds if there's no varint or an
+	// invalid varint (eg, a single 0xff byte). Correcting will add a bit of
+	// overhead. We could avoid that overhead whenever len(data) >=
+	// binary.MaxVarint32?
+
 	var v uint32
 	var n int
 	ptr := unsafe.Pointer(&data[0])
@@ -1473,19 +1531,21 @@ func ReadBatch(repr []byte) (r BatchReader, count uint32) {
 	return repr[batchHeaderLen:], count
 }
 
-// Next returns the next entry in this batch. The final return value is false
+// Next returns the next entry in this batch, if there is one. If the reader has
-// if the batch is corrupt. The end of batch is reached when len(r)==0.
+// reached the end of the batch, Next returns ok=false and a nil error. If the
-func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, ok bool) {
+// batch is corrupt and the next entry is illegible, Next returns ok=false and a
+// non-nil error.
+func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, ok bool, err error) {
 	if len(*r) == 0 {
-		return 0, nil, nil, false
+		return 0, nil, nil, false, nil
 	}
 	kind = InternalKeyKind((*r)[0])
 	if kind > InternalKeyKindMax {
-		return 0, nil, nil, false
+		return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "invalid key kind 0x%x", (*r)[0])
 	}
 	*r, ukey, ok = batchDecodeStr((*r)[1:])
 	if !ok {
-		return 0, nil, nil, false
+		return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "decoding user key")
 	}
 	switch kind {
 	case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete,
@@ -1493,10 +1553,10 @@ func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, o
 		InternalKeyKindDeleteSized:
 		*r, value, ok = batchDecodeStr(*r)
 		if !ok {
-			return 0, nil, nil, false
+			return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "decoding %s value", kind)
 		}
 	}
-	return kind, ukey, value, true
+	return kind, ukey, value, true, nil
 }
 
 // Note: batchIter mirrors the implementation of flushableBatchIter. Keep the
@@ -1698,7 +1758,7 @@ var _ flushable = (*flushableBatch)(nil)
 // interface. This allows the batch to act like a memtable and be placed in the
 // queue of flushable memtables. Note that the flushable batch takes ownership
 // of the batch data.
-func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
+func newFlushableBatch(batch *Batch, comparer *Comparer) (*flushableBatch, error) {
 	b := &flushableBatch{
 		data:      batch.data,
 		cmp:       comparer.Compare,
@@ -1719,8 +1779,11 @@ func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
 		var index uint32
 		for iter := BatchReader(b.data[batchHeaderLen:]); len(iter) > 0; index++ {
 			offset := uintptr(unsafe.Pointer(&iter[0])) - uintptr(unsafe.Pointer(&b.data[0]))
-			kind, key, _, ok := iter.Next()
+			kind, key, _, ok, err := iter.Next()
 			if !ok {
+				if err != nil {
+					return nil, err
+				}
 				break
 			}
 			entry := flushableBatchEntry{
@@ -1792,7 +1855,7 @@ func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
 		}
 		fragmentRangeKeys(frag, it, len(rangeKeyOffsets))
 	}
-	return b
+	return b, nil
 }
 
 func (b *flushableBatch) setSeqNum(seqNum uint64) {
@@ -1841,6 +1904,7 @@ func (b *flushableBatch) Swap(i, j int) {
 	b.offsets[i], b.offsets[j] = b.offsets[j], b.offsets[i]
 }
 
+// newIter is part of the flushable interface.
 func (b *flushableBatch) newIter(o *IterOptions) internalIterator {
 	return &flushableBatchIter{
 		batch:   b,
@@ -1853,6 +1917,7 @@ func (b *flushableBatch) newIter(o *IterOptions) internalIterator {
 	}
 }
 
+// newFlushIter is part of the flushable interface.
 func (b *flushableBatch) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
 	return &flushFlushableBatchIter{
 		flushableBatchIter: flushableBatchIter{
@@ -1866,6 +1931,7 @@ func (b *flushableBatch) newFlushIter(o *IterOptions, bytesFlushed *uint64) inte
 	}
 }
 
+// newRangeDelIter is part of the flushable interface.
 func (b *flushableBatch) newRangeDelIter(o *IterOptions) keyspan.FragmentIterator {
 	if len(b.tombstones) == 0 {
 		return nil
@@ -1873,6 +1939,7 @@ func (b *flushableBatch) newRangeDelIter(o *IterOptions) keyspan.FragmentIterato
 	return keyspan.NewIter(b.cmp, b.tombstones)
 }
 
+// newRangeKeyIter is part of the flushable interface.
 func (b *flushableBatch) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator {
 	if len(b.rangeKeys) == 0 {
 		return nil
@@ -1880,17 +1947,23 @@ func (b *flushableBatch) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterato
 	return keyspan.NewIter(b.cmp, b.rangeKeys)
 }
 
+// containsRangeKeys is part of the flushable interface.
 func (b *flushableBatch) containsRangeKeys() bool { return len(b.rangeKeys) > 0 }
 
+// inuseBytes is part of the flushable interface.
 func (b *flushableBatch) inuseBytes() uint64 {
 	return uint64(len(b.data) - batchHeaderLen)
 }
 
+// totalBytes is part of the flushable interface.
 func (b *flushableBatch) totalBytes() uint64 {
 	return uint64(cap(b.data))
 }
 
+// readyForFlush is part of the flushable interface.
 func (b *flushableBatch) readyForFlush() bool {
+	// A flushable batch is always ready for flush; it must be flushed together
+	// with the previous memtable.
 	return true
 }
 
@@ -2208,7 +2281,10 @@ func batchSort(
 		rangeKeyIter := b.newRangeKeyIter(nil, math.MaxUint64)
 		return pointIter, rangeDelIter, rangeKeyIter
 	}
-	f := newFlushableBatch(b, b.db.opts.Comparer)
+	f, err := newFlushableBatch(b, b.db.opts.Comparer)
+	if err != nil {
+		panic(err)
+	}
 	return f.newIter(nil), f.newRangeDelIter(nil), f.newRangeKeyIter(nil)
 }
 

vendor/github.com/cockroachdb/pebble/checkpoint.go

@@ -135,9 +135,6 @@ func mkdirAllAndSyncParents(fs vfs.FS, destDir string) (vfs.File, error) {
 // space overhead for a checkpoint if hard links are disabled. Also beware that
 // even if hard links are used, the space overhead for the checkpoint will
 // increase over time as the DB performs compactions.
-//
-// TODO(bananabrick): Test checkpointing of virtual sstables once virtual
-// sstables is running e2e.
 func (d *DB) Checkpoint(
 	destDir string, opts ...CheckpointOption,
 ) (
@@ -187,12 +184,12 @@ func (d *DB) Checkpoint(
 	// file number.
 	memQueue := d.mu.mem.queue
 	current := d.mu.versions.currentVersion()
-	formatVers := d.mu.formatVers.vers
+	formatVers := d.FormatMajorVersion()
 	manifestFileNum := d.mu.versions.manifestFileNum
 	manifestSize := d.mu.versions.manifest.Size()
 	optionsFileNum := d.optionsFileNum
 	virtualBackingFiles := make(map[base.DiskFileNum]struct{})
-	for diskFileNum := range d.mu.versions.fileBackingMap {
+	for diskFileNum := range d.mu.versions.backingState.fileBackingMap {
 		virtualBackingFiles[diskFileNum] = struct{}{}
 	}
 	// Release the manifest and DB.mu so we don't block other operations on
@@ -215,11 +212,7 @@ func (d *DB) Checkpoint(
 		}
 		if ckErr != nil {
 			// Attempt to cleanup on error.
-			paths, _ := fs.List(destDir)
+			_ = fs.RemoveAll(destDir)
-			for _, path := range paths {
-				_ = fs.Remove(path)
-			}
-			_ = fs.Remove(destDir)
 		}
 	}()
 	dir, ckErr = mkdirAllAndSyncParents(fs, destDir)

vendor/github.com/cockroachdb/pebble/cleaner.go

@@ -4,7 +4,18 @@
 
 package pebble
 
-import "github.com/cockroachdb/pebble/internal/base"
+import (
+	"context"
+	"runtime/pprof"
+	"sync"
+	"time"
+
+	"github.com/cockroachdb/errors/oserror"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/tokenbucket"
+)
 
 // Cleaner exports the base.Cleaner type.
 type Cleaner = base.Cleaner
@@ -14,3 +25,271 @@ type DeleteCleaner = base.DeleteCleaner
 
 // ArchiveCleaner exports the base.ArchiveCleaner type.
 type ArchiveCleaner = base.ArchiveCleaner
+
+type cleanupManager struct {
+	opts            *Options
+	objProvider     objstorage.Provider
+	onTableDeleteFn func(fileSize uint64)
+	deletePacer     *deletionPacer
+
+	// jobsCh is used as the cleanup job queue.
+	jobsCh chan *cleanupJob
+	// waitGroup is used to wait for the background goroutine to exit.
+	waitGroup sync.WaitGroup
+
+	mu struct {
+		sync.Mutex
+		// totalJobs is the total number of enqueued jobs (completed or in progress).
+		totalJobs              int
+		completedJobs          int
+		completedJobsCond      sync.Cond
+		jobsQueueWarningIssued bool
+	}
+}
+
+// We can queue this many jobs before we have to block EnqueueJob.
+const jobsQueueDepth = 1000
+
+// obsoleteFile holds information about a file that needs to be deleted soon.
+type obsoleteFile struct {
+	dir      string
+	fileNum  base.DiskFileNum
+	fileType fileType
+	fileSize uint64
+}
+
+type cleanupJob struct {
+	jobID         int
+	obsoleteFiles []obsoleteFile
+}
+
+// openCleanupManager creates a cleanupManager and starts its background goroutine.
+// The cleanupManager must be Close()d.
+func openCleanupManager(
+	opts *Options,
+	objProvider objstorage.Provider,
+	onTableDeleteFn func(fileSize uint64),
+	getDeletePacerInfo func() deletionPacerInfo,
+) *cleanupManager {
+	cm := &cleanupManager{
+		opts:            opts,
+		objProvider:     objProvider,
+		onTableDeleteFn: onTableDeleteFn,
+		deletePacer:     newDeletionPacer(time.Now(), int64(opts.TargetByteDeletionRate), getDeletePacerInfo),
+		jobsCh:          make(chan *cleanupJob, jobsQueueDepth),
+	}
+	cm.mu.completedJobsCond.L = &cm.mu.Mutex
+	cm.waitGroup.Add(1)
+
+	go func() {
+		pprof.Do(context.Background(), gcLabels, func(context.Context) {
+			cm.mainLoop()
+		})
+	}()
+
+	return cm
+}
+
+// Close stops the background goroutine, waiting until all queued jobs are completed.
+// Delete pacing is disabled for the remaining jobs.
+func (cm *cleanupManager) Close() {
+	close(cm.jobsCh)
+	cm.waitGroup.Wait()
+}
+
+// EnqueueJob adds a cleanup job to the manager's queue.
+func (cm *cleanupManager) EnqueueJob(jobID int, obsoleteFiles []obsoleteFile) {
+	job := &cleanupJob{
+		jobID:         jobID,
+		obsoleteFiles: obsoleteFiles,
+	}
+
+	// Report deleted bytes to the pacer, which can use this data to potentially
+	// increase the deletion rate to keep up. We want to do this at enqueue time
+	// rather than when we get to the job, otherwise the reported bytes will be
+	// subject to the throttling rate which defeats the purpose.
+	var pacingBytes uint64
+	for _, of := range obsoleteFiles {
+		if cm.needsPacing(of.fileType, of.fileNum) {
+			pacingBytes += of.fileSize
+		}
+	}
+	if pacingBytes > 0 {
+		cm.deletePacer.ReportDeletion(time.Now(), pacingBytes)
+	}
+
+	cm.mu.Lock()
+	cm.mu.totalJobs++
+	cm.maybeLogLocked()
+	cm.mu.Unlock()
+
+	if invariants.Enabled && len(cm.jobsCh) >= cap(cm.jobsCh)-2 {
+		panic("cleanup jobs queue full")
+	}
+
+	cm.jobsCh <- job
+}
+
+// Wait until the completion of all jobs that were already queued.
+//
+// Does not wait for jobs that are enqueued during the call.
+//
+// Note that DB.mu should not be held while calling this method; the background
+// goroutine needs to acquire DB.mu to update deleted table metrics.
+func (cm *cleanupManager) Wait() {
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+	n := cm.mu.totalJobs
+	for cm.mu.completedJobs < n {
+		cm.mu.completedJobsCond.Wait()
+	}
+}
+
+// mainLoop runs the manager's background goroutine.
+func (cm *cleanupManager) mainLoop() {
+	defer cm.waitGroup.Done()
+
+	var tb tokenbucket.TokenBucket
+	// Use a token bucket with 1 token / second refill rate and 1 token burst.
+	tb.Init(1.0, 1.0)
+	for job := range cm.jobsCh {
+		for _, of := range job.obsoleteFiles {
+			if of.fileType != fileTypeTable {
+				path := base.MakeFilepath(cm.opts.FS, of.dir, of.fileType, of.fileNum)
+				cm.deleteObsoleteFile(of.fileType, job.jobID, path, of.fileNum, of.fileSize)
+			} else {
+				cm.maybePace(&tb, of.fileType, of.fileNum, of.fileSize)
+				cm.onTableDeleteFn(of.fileSize)
+				cm.deleteObsoleteObject(fileTypeTable, job.jobID, of.fileNum)
+			}
+		}
+		cm.mu.Lock()
+		cm.mu.completedJobs++
+		cm.mu.completedJobsCond.Broadcast()
+		cm.maybeLogLocked()
+		cm.mu.Unlock()
+	}
+}
+
+func (cm *cleanupManager) needsPacing(fileType base.FileType, fileNum base.DiskFileNum) bool {
+	if fileType != fileTypeTable {
+		return false
+	}
+	meta, err := cm.objProvider.Lookup(fileType, fileNum)
+	if err != nil {
+		// The object was already removed from the provider; we won't actually
+		// delete anything, so we don't need to pace.
+		return false
+	}
+	// Don't throttle deletion of remote objects.
+	return !meta.IsRemote()
+}
+
+// maybePace sleeps before deleting an object if appropriate. It is always
+// called from the background goroutine.
+func (cm *cleanupManager) maybePace(
+	tb *tokenbucket.TokenBucket, fileType base.FileType, fileNum base.DiskFileNum, fileSize uint64,
+) {
+	if !cm.needsPacing(fileType, fileNum) {
+		return
+	}
+
+	tokens := cm.deletePacer.PacingDelay(time.Now(), fileSize)
+	if tokens == 0.0 {
+		// The token bucket might be in debt; it could make us wait even for 0
+		// tokens. We don't want that if the pacer decided throttling should be
+		// disabled.
+		return
+	}
+	// Wait for tokens. We use a token bucket instead of sleeping outright because
+	// the token bucket accumulates up to one second of unused tokens.
+	for {
+		ok, d := tb.TryToFulfill(tokenbucket.Tokens(tokens))
+		if ok {
+			break
+		}
+		time.Sleep(d)
+	}
+}
+
+// deleteObsoleteFile deletes a (non-object) file that is no longer needed.
+func (cm *cleanupManager) deleteObsoleteFile(
+	fileType fileType, jobID int, path string, fileNum base.DiskFileNum, fileSize uint64,
+) {
+	// TODO(peter): need to handle this error, probably by re-adding the
+	// file that couldn't be deleted to one of the obsolete slices map.
+	err := cm.opts.Cleaner.Clean(cm.opts.FS, fileType, path)
+	if oserror.IsNotExist(err) {
+		return
+	}
+
+	switch fileType {
+	case fileTypeLog:
+		cm.opts.EventListener.WALDeleted(WALDeleteInfo{
+			JobID:   jobID,
+			Path:    path,
+			FileNum: fileNum.FileNum(),
+			Err:     err,
+		})
+	case fileTypeManifest:
+		cm.opts.EventListener.ManifestDeleted(ManifestDeleteInfo{
+			JobID:   jobID,
+			Path:    path,
+			FileNum: fileNum.FileNum(),
+			Err:     err,
+		})
+	case fileTypeTable:
+		panic("invalid deletion of object file")
+	}
+}
+
+func (cm *cleanupManager) deleteObsoleteObject(
+	fileType fileType, jobID int, fileNum base.DiskFileNum,
+) {
+	if fileType != fileTypeTable {
+		panic("not an object")
+	}
+
+	var path string
+	meta, err := cm.objProvider.Lookup(fileType, fileNum)
+	if err != nil {
+		path = "<nil>"
+	} else {
+		path = cm.objProvider.Path(meta)
+		err = cm.objProvider.Remove(fileType, fileNum)
+	}
+	if cm.objProvider.IsNotExistError(err) {
+		return
+	}
+
+	switch fileType {
+	case fileTypeTable:
+		cm.opts.EventListener.TableDeleted(TableDeleteInfo{
+			JobID:   jobID,
+			Path:    path,
+			FileNum: fileNum.FileNum(),
+			Err:     err,
+		})
+	}
+}
+
+// maybeLogLocked issues a log if the job queue gets 75% full and issues a log
+// when the job queue gets back to less than 10% full.
+//
+// Must be called with cm.mu locked.
+func (cm *cleanupManager) maybeLogLocked() {
+	const highThreshold = jobsQueueDepth * 3 / 4
+	const lowThreshold = jobsQueueDepth / 10
+
+	jobsInQueue := cm.mu.totalJobs - cm.mu.completedJobs
+
+	if !cm.mu.jobsQueueWarningIssued && jobsInQueue > highThreshold {
+		cm.mu.jobsQueueWarningIssued = true
+		cm.opts.Logger.Infof("cleanup falling behind; job queue has over %d jobs", highThreshold)
+	}
+
+	if cm.mu.jobsQueueWarningIssued && jobsInQueue < lowThreshold {
+		cm.mu.jobsQueueWarningIssued = false
+		cm.opts.Logger.Infof("cleanup back to normal; job queue has under %d jobs", lowThreshold)
+	}
+}

vendor/github.com/cockroachdb/pebble/commit.go

@@ -9,7 +9,6 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
-	"unsafe"
 
 	"github.com/cockroachdb/pebble/record"
 )
@@ -39,7 +38,7 @@ type commitQueue struct {
 	// power of 2. A slot is in use until *both* the tail index has moved beyond
 	// it and the slot value has been set to nil. The slot value is set to nil
 	// atomically by the consumer and read atomically by the producer.
-	slots [record.SyncConcurrency]unsafe.Pointer
+	slots [record.SyncConcurrency]atomic.Pointer[Batch]
 }
 
 const dequeueBits = 32
@@ -67,8 +66,8 @@ func (q *commitQueue) enqueue(b *Batch) {
 	}
 	slot := &q.slots[head&uint32(len(q.slots)-1)]
 
-	// Check if the head slot has been released by dequeue.
+	// Check if the head slot has been released by dequeueApplied.
-	for atomic.LoadPointer(slot) != nil {
+	for slot.Load() != nil {
 		// Another goroutine is still cleaning up the tail, so the queue is
 		// actually still full. We spin because this should resolve itself
 		// momentarily.
@@ -76,14 +75,18 @@ func (q *commitQueue) enqueue(b *Batch) {
 	}
 
 	// The head slot is free, so we own it.
-	atomic.StorePointer(slot, unsafe.Pointer(b))
+	slot.Store(b)
 
-	// Increment head. This passes ownership of slot to dequeue and acts as a
+	// Increment head. This passes ownership of slot to dequeueApplied and acts as a
 	// store barrier for writing the slot.
 	q.headTail.Add(1 << dequeueBits)
 }
 
-func (q *commitQueue) dequeue() *Batch {
+// dequeueApplied removes the earliest enqueued Batch, if it is applied.
+//
+// Returns nil if the commit queue is empty or the earliest Batch is not yet
+// applied.
+func (q *commitQueue) dequeueApplied() *Batch {
 	for {
 		ptrs := q.headTail.Load()
 		head, tail := q.unpack(ptrs)
@@ -93,7 +96,7 @@ func (q *commitQueue) dequeue() *Batch {
 		}
 
 		slot := &q.slots[tail&uint32(len(q.slots)-1)]
-		b := (*Batch)(atomic.LoadPointer(slot))
+		b := slot.Load()
 		if b == nil || !b.applied.Load() {
 			// The batch is not ready to be dequeued, or another goroutine has
 			// already dequeued it.
@@ -109,7 +112,7 @@ func (q *commitQueue) dequeue() *Batch {
 			// Tell enqueue that we're done with this slot. Zeroing the slot is also
 			// important so we don't leave behind references that could keep this object
 			// live longer than necessary.
-			atomic.StorePointer(slot, nil)
+			slot.Store(nil)
 			// At this point enqueue owns the slot.
 			return b
 		}
@@ -239,6 +242,7 @@ type commitPipeline struct {
 	// logSyncQSem are used for this reservation.
 	commitQueueSem chan struct{}
 	logSyncQSem    chan struct{}
+	ingestSem      chan struct{}
 	// The mutex to use for synchronizing access to logSeqNum and serializing
 	// calls to commitEnv.write().
 	mu sync.Mutex
@@ -263,6 +267,7 @@ func newCommitPipeline(env commitEnv) *commitPipeline {
 		// and sync the WAL.
 		commitQueueSem: make(chan struct{}, record.SyncConcurrency-1),
 		logSyncQSem:    make(chan struct{}, record.SyncConcurrency-1),
+		ingestSem:      make(chan struct{}, 1),
 	}
 	return p
 }
@@ -471,13 +476,13 @@ func (p *commitPipeline) publish(b *Batch) {
 
 	// Loop dequeuing applied batches from the pending queue. If our batch was
 	// the head of the pending queue we are guaranteed that either we'll publish
-	// it or someone else will dequeue and publish it. If our batch is not the
+	// it or someone else will dequeueApplied and publish it. If our batch is not the
-	// head of the queue then either we'll dequeue applied batches and reach our
+	// head of the queue then either we'll dequeueApplied applied batches and reach our
 	// batch or there is an unapplied batch blocking us. When that unapplied
 	// batch applies it will go through the same process and publish our batch
 	// for us.
 	for {
-		t := p.pending.dequeue()
+		t := p.pending.dequeueApplied()
 		if t == nil {
 			// Wait for another goroutine to publish us. We might also be waiting for
 			// the WAL sync to finish.

vendor/github.com/cockroachdb/pebble/compaction_iter.go

@@ -15,9 +15,9 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/bytealloc"
-	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/rangekey"
+	"github.com/cockroachdb/redact"
 )
 
 // compactionIter provides a forward-only iterator that encapsulates the logic
@@ -207,14 +207,6 @@ type compactionIter struct {
 	// its elision. This field only applies to point keys, and not to range
 	// deletions or range keys.
 	//
-	// snapshotPinned is also used to set the forceObsolete value in the call to
-	// Writer.AddWithForceObsolete. Note that in that call, it is sufficient to
-	// mark all keys obsoleted by RANGEDELs as forceObsolete=true and that the
-	// implementation of Writer.AddWithForceObsolete will itself discover other
-	// causes of obsolescence. We mention this since in the presence of MERGE,
-	// obsolescence due to multiple keys at the same user key is not fully
-	// represented by snapshotPinned=true:
-	//
 	// For MERGE, it is possible that doing the merge is interrupted even when
 	// the next point key is in the same stripe. This can happen if the loop in
 	// mergeNext gets interrupted by sameStripeNonSkippable.
@@ -222,6 +214,21 @@ type compactionIter struct {
 	// SET/MERGE/DEL with the same seqnum, so the RANGEDEL does not necessarily
 	// delete the subsequent SET/MERGE/DEL keys.
 	snapshotPinned bool
+	// forceObsoleteDueToRangeDel is set to true in a subset of the cases that
+	// snapshotPinned is true. This value is true when the point is obsolete due
+	// to a RANGEDEL but could not be deleted due to a snapshot.
+	//
+	// NB: it may seem that the additional cases that snapshotPinned captures
+	// are harmless in that they can also be used to mark a point as obsolete
+	// (it is merely a duplication of some logic that happens in
+	// Writer.AddWithForceObsolete), but that is not quite accurate as of this
+	// writing -- snapshotPinned originated in stats collection and for a
+	// sequence MERGE, SET, where the MERGE cannot merge with the (older) SET
+	// due to a snapshot, the snapshotPinned value for the SET is true.
+	//
+	// TODO(sumeer,jackson): improve the logic of snapshotPinned and reconsider
+	// whether we need forceObsoleteDueToRangeDel.
+	forceObsoleteDueToRangeDel bool
 	// The index of the snapshot for the current key within the snapshots slice.
 	curSnapshotIdx    int
 	curSnapshotSeqNum uint64
@@ -249,10 +256,12 @@ type compactionIter struct {
 	// The fragmented range keys.
 	rangeKeys []keyspan.Span
 	// Byte allocator for the tombstone keys.
-	alloc               bytealloc.A
+	alloc                                  bytealloc.A
-	allowZeroSeqNum     bool
+	allowZeroSeqNum                        bool
-	elideTombstone      func(key []byte) bool
+	elideTombstone                         func(key []byte) bool
-	elideRangeTombstone func(start, end []byte) bool
+	elideRangeTombstone                    func(start, end []byte) bool
+	ineffectualSingleDeleteCallback        func(userKey []byte)
+	singleDeleteInvariantViolationCallback func(userKey []byte)
 	// The on-disk format major version. This informs the types of keys that
 	// may be written to disk during a compaction.
 	formatVersion FormatMajorVersion
@@ -274,20 +283,24 @@ func newCompactionIter(
 	allowZeroSeqNum bool,
 	elideTombstone func(key []byte) bool,
 	elideRangeTombstone func(start, end []byte) bool,
+	ineffectualSingleDeleteCallback func(userKey []byte),
+	singleDeleteInvariantViolationCallback func(userKey []byte),
 	formatVersion FormatMajorVersion,
 ) *compactionIter {
 	i := &compactionIter{
-		equal:               equal,
+		equal:                                  equal,
-		merge:               merge,
+		merge:                                  merge,
-		iter:                iter,
+		iter:                                   iter,
-		snapshots:           snapshots,
+		snapshots:                              snapshots,
-		frontiers:           frontiers{cmp: cmp},
+		frontiers:                              frontiers{cmp: cmp},
-		rangeDelFrag:        rangeDelFrag,
+		rangeDelFrag:                           rangeDelFrag,
-		rangeKeyFrag:        rangeKeyFrag,
+		rangeKeyFrag:                           rangeKeyFrag,
-		allowZeroSeqNum:     allowZeroSeqNum,
+		allowZeroSeqNum:                        allowZeroSeqNum,
-		elideTombstone:      elideTombstone,
+		elideTombstone:                         elideTombstone,
-		elideRangeTombstone: elideRangeTombstone,
+		elideRangeTombstone:                    elideRangeTombstone,
-		formatVersion:       formatVersion,
+		ineffectualSingleDeleteCallback:        ineffectualSingleDeleteCallback,
+		singleDeleteInvariantViolationCallback: singleDeleteInvariantViolationCallback,
+		formatVersion:                          formatVersion,
 	}
 	i.rangeDelFrag.Cmp = cmp
 	i.rangeDelFrag.Format = formatKey
@@ -326,7 +339,7 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 		return nil, nil
 	}
 
-	// Prior to this call to `Next()` we are in one of three situations with
+	// Prior to this call to `Next()` we are in one of four situations with
 	// respect to `iterKey` and related state:
 	//
 	// - `!skip && pos == iterPosNext`: `iterKey` is already at the next key.
@@ -335,12 +348,34 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 	//   snapshot stripe.
 	// - `skip && pos == iterPosCurForward`: We are at the key that has been returned.
 	//   To move forward we skip skippable entries in the stripe.
+	// - `skip && pos == iterPosNext && i.iterStripeChange == sameStripeNonSkippable`:
+	//    This case may occur when skipping within a snapshot stripe and we
+	//    encounter either:
+	//      a) an invalid key kind; The previous call will have returned
+	//         whatever key it was processing and deferred handling of the
+	//         invalid key to this invocation of Next(). We're responsible for
+	//         ignoring skip=true and falling into the invalid key kind case
+	//         down below.
+	//      b) an interleaved range delete; This is a wart of the current code
+	//         structure. While skipping within a snapshot stripe, a range
+	//         delete interleaved at its start key and sequence number
+	//         interrupts the sequence of point keys. After we return the range
+	//         delete to the caller, we need to pick up skipping at where we
+	//         left off, so we preserve skip=true.
+	//    TODO(jackson): This last case is confusing and can be removed if we
+	//    interleave range deletions at the maximal sequence number using the
+	//    keyspan interleaving iterator. This is the treatment given to range
+	//    keys today.
 	if i.pos == iterPosCurForward {
 		if i.skip {
 			i.skipInStripe()
 		} else {
 			i.nextInStripe()
 		}
+	} else if i.skip {
+		if i.iterStripeChange != sameStripeNonSkippable {
+			panic(errors.AssertionFailedf("compaction iterator has skip=true, but iterator is at iterPosNext"))
+		}
 	}
 
 	i.pos = iterPosCurForward
@@ -394,6 +429,10 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			return &i.key, i.value
 		}
 
+		// TODO(sumeer): we could avoid calling Covers if i.iterStripeChange ==
+		// sameStripeSameKey since that check has already been done in
+		// nextInStripeHelper. However, we also need to handle the case of
+		// CoversInvisibly below.
 		if cover := i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum); cover == keyspan.CoversVisibly {
 			// A pending range deletion deletes this key. Skip it.
 			i.saveKey()
@@ -410,6 +449,9 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			// key is in the same snapshot stripe. Hence, snapshotPinned is by
 			// definition false in those cases.
 			i.snapshotPinned = true
+			i.forceObsoleteDueToRangeDel = true
+		} else {
+			i.forceObsoleteDueToRangeDel = false
 		}
 
 		switch i.iterKey.Kind() {
@@ -419,7 +461,17 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 					// If we're at the last snapshot stripe and the tombstone
 					// can be elided skip skippable keys in the same stripe.
 					i.saveKey()
-					i.skipInStripe()
+					if i.key.Kind() == InternalKeyKindSingleDelete {
+						i.skipDueToSingleDeleteElision()
+					} else {
+						i.skipInStripe()
+						if !i.skip && i.iterStripeChange != newStripeNewKey {
+							panic(errors.AssertionFailedf("pebble: skipInStripe in last stripe disabled skip without advancing to new key"))
+						}
+					}
+					if i.iterStripeChange == newStripeSameKey {
+						panic(errors.AssertionFailedf("pebble: skipInStripe in last stripe found a new stripe within the same key"))
+					}
 					continue
 				} else {
 					// We're not at the last snapshot stripe, so the tombstone
@@ -446,8 +498,14 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			case InternalKeyKindSingleDelete:
 				if i.singleDeleteNext() {
 					return &i.key, i.value
+				} else if i.err != nil {
+					return nil, nil
 				}
 				continue
+
+			default:
+				panic(errors.AssertionFailedf(
+					"unexpected kind %s", redact.SafeString(i.iterKey.Kind().String())))
 			}
 
 		case InternalKeyKindSet, InternalKeyKindSetWithDelete:
@@ -457,6 +515,9 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			// preserving the original value, and potentially mutating the key
 			// kind.
 			i.setNext()
+			if i.err != nil {
+				return nil, nil
+			}
 			return &i.key, i.value
 
 		case InternalKeyKindMerge:
@@ -473,7 +534,15 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			if i.err == nil {
 				// includesBase is true whenever we've transformed the MERGE record
 				// into a SET.
-				includesBase := i.key.Kind() == InternalKeyKindSet
+				var includesBase bool
+				switch i.key.Kind() {
+				case InternalKeyKindSet, InternalKeyKindSetWithDelete:
+					includesBase = true
+				case InternalKeyKindMerge:
+				default:
+					panic(errors.AssertionFailedf(
+						"unexpected kind %s", redact.SafeString(i.key.Kind().String())))
+				}
 				i.value, needDelete, i.valueCloser, i.err = finishValueMerger(valueMerger, includesBase)
 			}
 			if i.err == nil {
@@ -502,6 +571,8 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			}
 			if i.err != nil {
 				i.valid = false
+				// TODO(sumeer): why is MarkCorruptionError only being called for
+				// MERGE?
 				i.err = base.MarkCorruptionError(i.err)
 			}
 			return nil, nil
@@ -541,10 +612,16 @@ func snapshotIndex(seq uint64, snapshots []uint64) (int, uint64) {
 	return index, snapshots[index]
 }
 
-// skipInStripe skips over skippable keys in the same stripe and user key.
+// skipInStripe skips over skippable keys in the same stripe and user key. It
+// may set i.err, in which case i.iterKey will be nil.
 func (i *compactionIter) skipInStripe() {
 	i.skip = true
+	// TODO(sumeer): we can avoid the overhead of calling i.rangeDelFrag.Covers,
+	// in this case of nextInStripe, since we are skipping all of them anyway.
 	for i.nextInStripe() == sameStripeSkippable {
+		if i.err != nil {
+			panic(i.err)
+		}
 	}
 	// Reset skip if we landed outside the original stripe. Otherwise, we landed
 	// in the same stripe on a non-skippable key. In that case we should preserve
@@ -582,11 +659,17 @@ const (
 // nextInStripe advances the iterator and returns one of the above const ints
 // indicating how its state changed.
 //
+// All sameStripeSkippable keys that are covered by a RANGEDEL will be skipped
+// and not returned.
+//
 // Calls to nextInStripe must be preceded by a call to saveKey to retain a
 // temporary reference to the original key, so that forward iteration can
 // proceed with a reference to the original key. Care should be taken to avoid
 // overwriting or mutating the saved key or value before they have been returned
 // to the caller of the exported function (i.e. the caller of Next, First, etc.)
+//
+// nextInStripe may set i.err, in which case the return value will be
+// newStripeNewKey, and i.iterKey will be nil.
 func (i *compactionIter) nextInStripe() stripeChangeType {
 	i.iterStripeChange = i.nextInStripeHelper()
 	return i.iterStripeChange
@@ -595,58 +678,72 @@ func (i *compactionIter) nextInStripe() stripeChangeType {
 // nextInStripeHelper is an internal helper for nextInStripe; callers should use
 // nextInStripe and not call nextInStripeHelper.
 func (i *compactionIter) nextInStripeHelper() stripeChangeType {
-	if !i.iterNext() {
+	origSnapshotIdx := i.curSnapshotIdx
-		return newStripeNewKey
+	for {
-	}
+		if !i.iterNext() {
-	key := i.iterKey
+			return newStripeNewKey
+		}
+		key := i.iterKey
 
-	// NB: The below conditional is an optimization to avoid a user key
+		if !i.equal(i.key.UserKey, key.UserKey) {
-	// comparison in many cases. Internal keys with the same user key are
+			i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
-	// ordered in (strictly) descending order by trailer. If the new key has a
+			return newStripeNewKey
-	// greater or equal trailer, or the previous key had a zero sequence number,
+		}
-	// the new key must have a new user key.
+
-	//
+		// If i.key and key have the same user key, then
-	// A couple things make these cases common:
+		//   1. i.key must not have had a zero sequence number (or it would've be the last
-	// - Sequence-number zeroing ensures ~all of the keys in L6 have a zero
+		//      key with its user key).
-	//   sequence number.
+		//   2. i.key must have a strictly larger sequence number
-	// - Ingested sstables' keys all adopt the same sequence number.
+		// There's an exception in that either key may be a range delete. Range
-	if i.keyTrailer <= base.InternalKeyZeroSeqnumMaxTrailer || key.Trailer >= i.keyTrailer {
+		// deletes may share a sequence number with a point key if the keys were
-		if invariants.Enabled && i.equal(i.key.UserKey, key.UserKey) {
+		// ingested together. Range keys may also share the sequence number if they
+		// were ingested, but range keys are interleaved into the compaction
+		// iterator's input iterator at the maximal sequence number so their
+		// original sequence number will not be observed here.
+		if prevSeqNum := base.SeqNumFromTrailer(i.keyTrailer); (prevSeqNum == 0 || prevSeqNum <= key.SeqNum()) &&
+			i.key.Kind() != InternalKeyKindRangeDelete && key.Kind() != InternalKeyKindRangeDelete {
 			prevKey := i.key
 			prevKey.Trailer = i.keyTrailer
-			panic(fmt.Sprintf("pebble: invariant violation: %s and %s out of order", key, prevKey))
+			panic(errors.AssertionFailedf("pebble: invariant violation: %s and %s out of order", prevKey, key))
 		}
+
 		i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
-		return newStripeNewKey
+		switch key.Kind() {
-	} else if !i.equal(i.key.UserKey, key.UserKey) {
+		case InternalKeyKindRangeDelete:
-		i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
+			// Range tombstones need to be exposed by the compactionIter to the upper level
-		return newStripeNewKey
+			// `compaction` object, so return them regardless of whether they are in the same
-	}
+			// snapshot stripe.
-	origSnapshotIdx := i.curSnapshotIdx
+			if i.curSnapshotIdx == origSnapshotIdx {
-	i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
+				return sameStripeNonSkippable
-	switch key.Kind() {
+			}
-	case InternalKeyKindRangeDelete:
+			return newStripeSameKey
-		// Range tombstones need to be exposed by the compactionIter to the upper level
+		case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
-		// `compaction` object, so return them regardless of whether they are in the same
+			// Range keys are interleaved at the max sequence number for a given user
-		// snapshot stripe.
+			// key, so we should not see any more range keys in this stripe.
-		if i.curSnapshotIdx == origSnapshotIdx {
+			panic("unreachable")
-			return sameStripeNonSkippable
+		case InternalKeyKindInvalid:
+			if i.curSnapshotIdx == origSnapshotIdx {
+				return sameStripeNonSkippable
+			}
+			return newStripeSameKey
+		case InternalKeyKindDelete, InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSingleDelete,
+			InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
+			// Fall through
+		default:
+			i.iterKey = nil
+			i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
+			i.valid = false
+			return newStripeNewKey
 		}
-		return newStripeSameKey
-	case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
-		// Range keys are interleaved at the max sequence number for a given user
-		// key, so we should not see any more range keys in this stripe.
-		panic("unreachable")
-	case InternalKeyKindInvalid:
 		if i.curSnapshotIdx == origSnapshotIdx {
-			return sameStripeNonSkippable
+			// Same snapshot.
+			if i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
+				continue
+			}
+			return sameStripeSkippable
 		}
 		return newStripeSameKey
 	}
-	if i.curSnapshotIdx == origSnapshotIdx {
-		return sameStripeSkippable
-	}
-	return newStripeSameKey
 }
 
 func (i *compactionIter) setNext() {
@@ -672,6 +769,10 @@ func (i *compactionIter) setNext() {
 
 	// Else, we continue to loop through entries in the stripe looking for a
 	// DEL. Note that we may stop *before* encountering a DEL, if one exists.
+	//
+	// NB: nextInStripe will skip sameStripeSkippable keys that are visibly
+	// covered by a RANGEDEL. This can include DELs -- this is fine since such
+	// DELs don't need to be combined with SET to make SETWITHDEL.
 	for {
 		switch i.nextInStripe() {
 		case newStripeNewKey, newStripeSameKey:
@@ -722,12 +823,16 @@ func (i *compactionIter) setNext() {
 			// We're still in the same stripe. If this is a
 			// DEL/SINGLEDEL/DELSIZED, we stop looking and emit a SETWITHDEL.
 			// Subsequent keys are eligible for skipping.
-			if i.iterKey.Kind() == InternalKeyKindDelete ||
+			switch i.iterKey.Kind() {
-				i.iterKey.Kind() == InternalKeyKindSingleDelete ||
+			case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
-				i.iterKey.Kind() == InternalKeyKindDeleteSized {
 				i.key.SetKind(InternalKeyKindSetWithDelete)
 				i.skip = true
 				return
+			case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
+				// Do nothing
+			default:
+				i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
+				i.valid = false
 			}
 		default:
 			panic("pebble: unexpected stripeChangeType: " + strconv.Itoa(int(i.iterStripeChange)))
@@ -747,13 +852,24 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
 			i.pos = iterPosNext
 			return i.iterStripeChange
 		}
+		if i.err != nil {
+			panic(i.err)
+		}
+		// NB: MERGE#10+RANGEDEL#9 stays a MERGE, since nextInStripe skips
+		// sameStripeSkippable keys that are visibly covered by a RANGEDEL. There
+		// may be MERGE#7 that is invisibly covered and will be preserved, but
+		// there is no risk that MERGE#10 and MERGE#7 will get merged in the
+		// future as the RANGEDEL still exists and will be used in user-facing
+		// reads that see MERGE#10, and will also eventually cause MERGE#7 to be
+		// deleted in a compaction.
 		key := i.iterKey
 		switch key.Kind() {
 		case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
 			// We've hit a deletion tombstone. Return everything up to this point and
 			// then skip entries until the next snapshot stripe. We change the kind
 			// of the result key to a Set so that it shadows keys in lower
-			// levels. That is, MERGE+DEL -> SET.
+			// levels. That is, MERGE+DEL -> SETWITHDEL.
+			//
 			// We do the same for SingleDelete since SingleDelete is only
 			// permitted (with deterministic behavior) for keys that have been
 			// set once since the last SingleDelete/Delete, so everything
@@ -766,21 +882,11 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
 			// single Set, and then merge in any following Sets, but that is
 			// complicated wrt code and unnecessary given the narrow permitted
 			// use of SingleDelete.
-			i.key.SetKind(InternalKeyKindSet)
+			i.key.SetKind(InternalKeyKindSetWithDelete)
 			i.skip = true
 			return sameStripeSkippable
 
 		case InternalKeyKindSet, InternalKeyKindSetWithDelete:
-			if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
-				// We change the kind of the result key to a Set so that it shadows
-				// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
-				// strictly necessary, but provides consistency with the behavior of
-				// MERGE+DEL.
-				i.key.SetKind(InternalKeyKindSet)
-				i.skip = true
-				return sameStripeSkippable
-			}
-
 			// We've hit a Set or SetWithDel value. Merge with the existing
 			// value and return. We change the kind of the resulting key to a
 			// Set so that it shadows keys in lower levels. That is:
@@ -795,16 +901,6 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
 			return sameStripeSkippable
 
 		case InternalKeyKindMerge:
-			if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
-				// We change the kind of the result key to a Set so that it shadows
-				// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
-				// strictly necessary, but provides consistency with the behavior of
-				// MERGE+DEL.
-				i.key.SetKind(InternalKeyKindSet)
-				i.skip = true
-				return sameStripeSkippable
-			}
-
 			// We've hit another Merge value. Merge with the existing value and
 			// continue looping.
 			i.err = valueMerger.MergeOlder(i.iterValue)
@@ -821,6 +917,16 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
 	}
 }
 
+// singleDeleteNext processes a SingleDelete point tombstone. A SingleDelete, or
+// SINGLEDEL, is unique in that it deletes exactly 1 internal key. It's a
+// performance optimization when the client knows a user key has not been
+// overwritten, allowing the elision of the tombstone earlier, avoiding write
+// amplification.
+//
+// singleDeleteNext returns a boolean indicating whether or not the caller
+// should yield the SingleDelete key to the consumer of the compactionIter. If
+// singleDeleteNext returns false, the caller may consume/elide the
+// SingleDelete.
 func (i *compactionIter) singleDeleteNext() bool {
 	// Save the current key.
 	i.saveKey()
@@ -829,26 +935,85 @@ func (i *compactionIter) singleDeleteNext() bool {
 
 	// Loop until finds a key to be passed to the next level.
 	for {
+		// If we find a key that can't be skipped, return true so that the
+		// caller yields the SingleDelete to the caller.
 		if i.nextInStripe() != sameStripeSkippable {
+			// This defers additional error checking regarding single delete
+			// invariants to the compaction where the keys with the same user key as
+			// the single delete are in the same stripe.
 			i.pos = iterPosNext
-			return true
+			return i.err == nil
 		}
-
+		if i.err != nil {
+			panic(i.err)
+		}
+		// INVARIANT: sameStripeSkippable.
 		key := i.iterKey
-		switch key.Kind() {
+		kind := key.Kind()
-		case InternalKeyKindDelete, InternalKeyKindMerge, InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
+		switch kind {
-			// We've hit a Delete, DeleteSized, Merge, SetWithDelete, transform
+		case InternalKeyKindDelete, InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
+			if (kind == InternalKeyKindDelete || kind == InternalKeyKindDeleteSized) &&
+				i.ineffectualSingleDeleteCallback != nil {
+				i.ineffectualSingleDeleteCallback(i.key.UserKey)
+			}
+			// We've hit a Delete, DeleteSized, SetWithDelete, transform
 			// the SingleDelete into a full Delete.
 			i.key.SetKind(InternalKeyKindDelete)
 			i.skip = true
 			return true
 
-		case InternalKeyKindSet:
+		case InternalKeyKindSet, InternalKeyKindMerge:
-			i.nextInStripe()
+			// This SingleDelete deletes the Set/Merge, and we can now elide the
+			// SingleDel as well. We advance past the Set and return false to
+			// indicate to the main compaction loop that we should NOT yield the
+			// current SingleDel key to the compaction loop.
+			//
+			// NB: singleDeleteNext was called with i.pos == iterPosCurForward, and
+			// after the call to nextInStripe, we are still at iterPosCurForward,
+			// since we are at the key after the Set/Merge that was single deleted.
+			change := i.nextInStripe()
+			switch change {
+			case sameStripeSkippable, newStripeSameKey:
+				// On the same user key.
+				nextKind := i.iterKey.Kind()
+				switch nextKind {
+				case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
+					if i.singleDeleteInvariantViolationCallback != nil {
+						// sameStripeSkippable keys returned by nextInStripe() are already
+						// known to not be covered by a RANGEDEL, so it is an invariant
+						// violation. The rare case is newStripeSameKey, where it is a
+						// violation if not covered by a RANGEDEL.
+						if change == sameStripeSkippable ||
+							i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) == keyspan.NoCover {
+							i.singleDeleteInvariantViolationCallback(i.key.UserKey)
+						}
+					}
+				case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete,
+					InternalKeyKindRangeDelete:
+				default:
+					panic(errors.AssertionFailedf(
+						"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
+				}
+			case sameStripeNonSkippable:
+				// No ability to check whether there is another Set/Merge below with
+				// the same user key.
+				//
+				// TODO(sumeer): once range deletions are interleaved at the maximal
+				// sequence number, this case will go away.
+			case newStripeNewKey:
+			default:
+				panic("unreachable")
+			}
 			i.valid = false
 			return false
 
 		case InternalKeyKindSingleDelete:
+			// Two single deletes met in a compaction. The first single delete is
+			// ineffectual.
+			if i.ineffectualSingleDeleteCallback != nil {
+				i.ineffectualSingleDeleteCallback(i.key.UserKey)
+			}
+			// Continue to apply the second single delete.
 			continue
 
 		default:
@@ -859,6 +1024,135 @@ func (i *compactionIter) singleDeleteNext() bool {
 	}
 }
 
+// skipDueToSingleDeleteElision is called when the SingleDelete is being
+// elided because it is in the final snapshot stripe and there are no keys
+// with the same user key in lower levels in the LSM (below the files in this
+// compaction).
+//
+// TODO(sumeer): the only difference between singleDeleteNext and
+// skipDueToSingleDeleteElision is the fact that the caller knows it will be
+// eliding the single delete in the latter case. There are some similar things
+// happening in both implementations. My first attempt at combining them into
+// a single method was hard to comprehend. Try again.
+func (i *compactionIter) skipDueToSingleDeleteElision() {
+	for {
+		stripeChange := i.nextInStripe()
+		if i.err != nil {
+			panic(i.err)
+		}
+		switch stripeChange {
+		case newStripeNewKey:
+			// The single delete is only now being elided, meaning it did not elide
+			// any keys earlier in its descent down the LSM. We stepped onto a new
+			// user key, meaning that even now at its moment of elision, it still
+			// hasn't elided any other keys. The single delete was ineffectual (a
+			// no-op).
+			if i.ineffectualSingleDeleteCallback != nil {
+				i.ineffectualSingleDeleteCallback(i.key.UserKey)
+			}
+			i.skip = false
+			return
+		case newStripeSameKey:
+			// This should be impossible. If we're eliding a single delete, we
+			// determined that the tombstone is in the final snapshot stripe, but we
+			// stepped into a new stripe of the same key.
+			panic(errors.AssertionFailedf("eliding single delete followed by same key in new stripe"))
+		case sameStripeNonSkippable:
+			// There's a key that we cannot skip. There are two possible cases:
+			//   a. The key is invalid. This is an error.
+			//   b. The key is a range deletion.
+			// The second case may also be an ineffectual single delete. However, it
+			// is possible that there is a SET that is at the same seqnum as the
+			// RANGEDEL, and so is not deleted by that RANGEDEL, and will be deleted
+			// by this single delete. So we cannot be certain that this is an
+			// ineffectual single delete.
+			//
+			// TODO(sumeer): the existing todo to interleave range deletions at the
+			// maximal sequence number will allow us to address this ambiguity.
+			//
+			// TODO(sumeer): by setting skip to true, the compactionIter is making a
+			// single delete stronger (like a del), which will hide bugs in the use of
+			// single delete.
+			i.skip = true
+			return
+		case sameStripeSkippable:
+			kind := i.iterKey.Kind()
+			switch kind {
+			case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete:
+				if i.ineffectualSingleDeleteCallback != nil {
+					i.ineffectualSingleDeleteCallback(i.key.UserKey)
+				}
+				switch kind {
+				case InternalKeyKindDelete, InternalKeyKindDeleteSized:
+					i.skipInStripe()
+					return
+				case InternalKeyKindSingleDelete:
+					// Repeat the same with this SingleDelete. We don't want to simply
+					// call skipInStripe(), since it increases the strength of the
+					// SingleDel, which hides bugs in the use of single delete.
+					continue
+				default:
+					panic(errors.AssertionFailedf(
+						"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
+				}
+			case InternalKeyKindSetWithDelete:
+				// The SingleDelete should behave like a Delete.
+				i.skipInStripe()
+				return
+			case InternalKeyKindSet, InternalKeyKindMerge:
+				// This SingleDelete deletes the Set/Merge, and we are eliding the
+				// SingleDel as well. Step to the next key (this is not deleted by the
+				// SingleDelete).
+				//
+				// NB: skipDueToSingleDeleteElision was called with i.pos ==
+				// iterPosCurForward, and after the call to nextInStripe, we are still
+				// at iterPosCurForward, since we are at the key after the Set/Merge
+				// that was single deleted.
+				change := i.nextInStripe()
+				if i.err != nil {
+					panic(i.err)
+				}
+				switch change {
+				case newStripeSameKey:
+					panic(errors.AssertionFailedf("eliding single delete followed by same key in new stripe"))
+				case newStripeNewKey:
+				case sameStripeSkippable:
+					// On the same key.
+					nextKind := i.iterKey.Kind()
+					switch nextKind {
+					case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
+						if i.singleDeleteInvariantViolationCallback != nil {
+							i.singleDeleteInvariantViolationCallback(i.key.UserKey)
+						}
+					case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete,
+						InternalKeyKindRangeDelete:
+					default:
+						panic(errors.AssertionFailedf(
+							"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
+					}
+				case sameStripeNonSkippable:
+					// No ability to check whether there is another Set/Merge below with
+					// the same user key.
+					//
+					// TODO(sumeer): once range deletions are interleaved at the maximal
+					// sequence number, this case will go away.
+				default:
+					panic("unreachable")
+				}
+				// Whether in same stripe or new stripe, this key is not consumed by
+				// the SingleDelete.
+				i.skip = false
+				return
+			default:
+				panic(errors.AssertionFailedf(
+					"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
+			}
+		default:
+			panic("unreachable")
+		}
+	}
+}
+
 // deleteSizedNext processes a DELSIZED point tombstone. Unlike ordinary DELs,
 // these tombstones carry a value that's a varint indicating the size of the
 // entry (len(key)+len(value)) that the tombstone is expected to delete.
@@ -881,28 +1175,137 @@ func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) {
 		i.value = i.valueBuf
 	}
 
-	// Loop through all the keys within this stripe that are skippable,
+	// Loop through all the keys within this stripe that are skippable.
-	// totalling the size of their keys and values.
 	i.pos = iterPosNext
-	var elidedSize uint64
 	for i.nextInStripe() == sameStripeSkippable {
-		elidedSize += uint64(len(i.iterKey.UserKey)) + uint64(len(i.iterValue))
+		if i.err != nil {
-
+			panic(i.err)
-		if i.iterKey.Kind() == InternalKeyKindDeleteSized {
+		}
-			// We encountered a DELSIZED that's deleted by the original
+		switch i.iterKey.Kind() {
-			// DELSIZED. These tombstones could've been intended to delete two
+		case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete:
-			// distinct values, eg:
+			// We encountered a tombstone (DEL, or DELSIZED) that's deleted by
+			// the original DELSIZED tombstone. This can happen in two cases:
+			//
+			// (1) These tombstones were intended to delete two distinct values,
+			//     and this DELSIZED has already dropped the relevant key. For
+			//     example:
 			//
 			//     a.DELSIZED.9   a.SET.7   a.DELSIZED.5   a.SET.4
 			//
-			// If a.DELSIZED.9 has already deleted a.SET.7, its size has already
+			//     If a.DELSIZED.9 has already deleted a.SET.7, its size has
-			// been zeroed out. In this case, we want to adopt the value of the
+			//     already been zeroed out. In this case, we want to adopt the
-			// DELSIZED with the lower sequence number, in case the a.SET.4 key
+			//     value of the DELSIZED with the lower sequence number, in
-			// has not yet been elided.
+			//     case the a.SET.4 key has not yet been elided.
+			//
+			// (2) This DELSIZED was missized. The user thought they were
+			//     deleting a key with this user key, but this user key had
+			//     already been deleted.
+			//
+			// We can differentiate these two cases by examining the length of
+			// the DELSIZED's value. A DELSIZED's value holds the size of both
+			// the user key and value that it intends to delete. For any user
+			// key with a length > 0, a DELSIZED that has not deleted a key must
+			// have a value with a length > 0.
+			//
+			// We treat both cases the same functionally, adopting the identity
+			// of the lower-sequence numbered tombstone. However in the second
+			// case, we also increment the stat counting missized tombstones.
+			if len(i.value) > 0 {
+				// The original DELSIZED key was missized. The key that the user
+				// thought they were deleting does not exist.
+				i.stats.countMissizedDels++
+			}
 			i.valueBuf = append(i.valueBuf[:0], i.iterValue...)
 			i.value = i.valueBuf
-			// Reset the elided total.
+			if i.iterKey.Kind() != InternalKeyKindDeleteSized {
-			elidedSize = 0
+				// Convert the DELSIZED to a DEL—The DEL/SINGLEDEL we're eliding
+				// may not have deleted the key(s) it was intended to yet. The
+				// ordinary DEL compaction heuristics are better suited at that,
+				// plus we don't want to count it as a missized DEL. We early
+				// exit in this case, after skipping the remainder of the
+				// snapshot stripe.
+				i.key.SetKind(InternalKeyKindDelete)
+				// NB: We skipInStripe now, rather than returning leaving
+				// i.skip=true and returning early, because Next() requires
+				// that i.skip=true only if i.iterPos = iterPosCurForward.
+				//
+				// Ignore any error caused by skipInStripe since it does not affect
+				// the key/value being returned here, and the next call to Next() will
+				// expose it.
+				i.skipInStripe()
+				return &i.key, i.value
+			}
+			// Continue, in case we uncover another DELSIZED or a key this
+			// DELSIZED deletes.
+
+		case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
+			// If the DELSIZED is value-less, it already deleted the key that it
+			// was intended to delete. This is possible with a sequence like:
+			//
+			//      DELSIZED.8     SET.7     SET.3
+			//
+			// The DELSIZED only describes the size of the SET.7, which in this
+			// case has already been elided. We don't count it as a missizing,
+			// instead converting the DELSIZED to a DEL. Skip the remainder of
+			// the snapshot stripe and return.
+			if len(i.value) == 0 {
+				i.key.SetKind(InternalKeyKindDelete)
+				// NB: We skipInStripe now, rather than returning leaving
+				// i.skip=true and returning early, because Next() requires
+				// that i.skip=true only if i.iterPos = iterPosCurForward.
+				//
+				// Ignore any error caused by skipInStripe since it does not affect
+				// the key/value being returned here, and the next call to Next() will
+				// expose it.
+				i.skipInStripe()
+				return &i.key, i.value
+			}
+			// The deleted key is not a DEL, DELSIZED, and the DELSIZED in i.key
+			// has a positive size.
+			expectedSize, n := binary.Uvarint(i.value)
+			if n != len(i.value) {
+				i.err = base.CorruptionErrorf("DELSIZED holds invalid value: %x", errors.Safe(i.value))
+				i.valid = false
+				return nil, nil
+			}
+			elidedSize := uint64(len(i.iterKey.UserKey)) + uint64(len(i.iterValue))
+			if elidedSize != expectedSize {
+				// The original DELSIZED key was missized. It's unclear what to
+				// do. The user-provided size was wrong, so it's unlikely to be
+				// accurate or meaningful. We could:
+				//
+				//   1. return the DELSIZED with the original user-provided size unmodified
+				//   2. return the DELZIZED with a zeroed size to reflect that a key was
+				//   elided, even if it wasn't the anticipated size.
+				//   3. subtract the elided size from the estimate and re-encode.
+				//   4. convert the DELSIZED into a value-less DEL, so that
+				//      ordinary DEL heuristics apply.
+				//
+				// We opt for (4) under the rationale that we can't rely on the
+				// user-provided size for accuracy, so ordinary DEL heuristics
+				// are safer.
+				i.stats.countMissizedDels++
+				i.key.SetKind(InternalKeyKindDelete)
+				i.value = i.valueBuf[:0]
+				// NB: We skipInStripe now, rather than returning leaving
+				// i.skip=true and returning early, because Next() requires
+				// that i.skip=true only if i.iterPos = iterPosCurForward.
+				//
+				// Ignore any error caused by skipInStripe since it does not affect
+				// the key/value being returned here, and the next call to Next() will
+				// expose it.
+				i.skipInStripe()
+				return &i.key, i.value
+			}
+			// NB: We remove the value regardless of whether the key was sized
+			// appropriately. The size encoded is 'consumed' the first time it
+			// meets a key that it deletes.
+			i.value = i.valueBuf[:0]
+
+		default:
+			i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
+			i.valid = false
+			return nil, nil
 		}
 	}
 	// Reset skip if we landed outside the original stripe. Otherwise, we landed
@@ -912,53 +1315,8 @@ func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) {
 	if i.iterStripeChange == newStripeNewKey || i.iterStripeChange == newStripeSameKey {
 		i.skip = false
 	}
-
+	if i.err != nil {
-	if elidedSize == 0 {
+		return nil, nil
-		// If we didn't find any keys that we could elide due to this tombstone
-		// (or at least none since the last DELSIZED encountered), we can return
-		// the key and our existing saved value verbatim.
-		return &i.key, i.value
-	}
-
-	// Some key(s) were elided as a result of this tombstone. Decode the
-	// tombstone's value to see if it matches the amount of data actually
-	// elided.
-	var v uint64
-	if len(i.value) > 0 {
-		var n int
-		v, n = binary.Uvarint(i.value)
-		if n != len(i.value) {
-			i.err = base.CorruptionErrorf("DELSIZED holds invalid value: %x", errors.Safe(i.value))
-			i.valid = false
-			return nil, nil
-		}
-	}
-
-	// If the user's prediction of the value size was correct, then from now on
-	// it's predicted to delete nothing. Return an empty value.
-	//
-	// NB: It's important that we still always return the tombstone, because
-	// DELSIZED's values are a best effort optimization, not a correctness
-	// mechanic. There is no guarantee that this key was only set once before
-	// the DELSIZED was written.
-
-	// If the tombstone carried a larger size than what was actually elided,
-	// it's unclear what to do. The user-provided size was wrong, so it's
-	// unlikely to be accurate or meaningful. We could:
-	//
-	//   1. return the DELSIZED with the original user-provided size unmodified
-	//   2. return the DELZIZED with a zeroed size to reflect that a key was
-	//      elided, even if it wasn't the anticipated size.
-	//   3. subtract the elided size from the estimate and re-encode.
-	//   4. convert the DELSIZED into a value-less DEL, so that ordinary DEL
-	//      heuristics apply.
-	//
-	// We opt for (4) under the rationale that we can't rely on the
-	// user-provided size for accuracy, so ordinary DEL heuristics are safer.
-	i.value = i.valueBuf[:0]
-	if elidedSize != v {
-		i.stats.countMissizedDels++
-		i.key.SetKind(InternalKeyKindDelete)
 	}
 	return &i.key, i.value
 }

vendor/github.com/cockroachdb/pebble/event.go

@@ -39,10 +39,11 @@ func formatFileNums(tables []TableInfo) string {
 	return buf.String()
 }
 
-// LevelInfo contains info pertaining to a partificular level.
+// LevelInfo contains info pertaining to a particular level.
 type LevelInfo struct {
 	Level  int
 	Tables []TableInfo
+	Score  float64
 }
 
 func (i LevelInfo) String() string {
@@ -51,8 +52,11 @@ func (i LevelInfo) String() string {
 
 // SafeFormat implements redact.SafeFormatter.
 func (i LevelInfo) SafeFormat(w redact.SafePrinter, _ rune) {
-	w.Printf("L%d [%s] (%s)", redact.Safe(i.Level), redact.Safe(formatFileNums(i.Tables)),
+	w.Printf("L%d [%s] (%s) Score=%.2f",
-		redact.Safe(humanize.Uint64(tablesTotalSize(i.Tables))))
+		redact.Safe(i.Level),
+		redact.Safe(formatFileNums(i.Tables)),
+		redact.Safe(humanize.Bytes.Uint64(tablesTotalSize(i.Tables))),
+		redact.Safe(i.Score))
 }
 
 // CompactionInfo contains the info for a compaction event.
@@ -75,6 +79,27 @@ type CompactionInfo struct {
 	TotalDuration time.Duration
 	Done          bool
 	Err           error
+
+	SingleLevelOverlappingRatio float64
+	MultiLevelOverlappingRatio  float64
+
+	// Annotations specifies additional info to appear in a compaction's event log line
+	Annotations compactionAnnotations
+}
+
+type compactionAnnotations []string
+
+// SafeFormat implements redact.SafeFormatter.
+func (ca compactionAnnotations) SafeFormat(w redact.SafePrinter, _ rune) {
+	if len(ca) == 0 {
+		return
+	}
+	for i := range ca {
+		if i != 0 {
+			w.Print(" ")
+		}
+		w.Printf("%s", redact.SafeString(ca[i]))
+	}
 }
 
 func (i CompactionInfo) String() string {
@@ -90,20 +115,25 @@ func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 	}
 
 	if !i.Done {
-		w.Printf("[JOB %d] compacting(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
+		w.Printf("[JOB %d] compacting(%s) ",
-		w.Print(levelInfos(i.Input))
+			redact.Safe(i.JobID),
+			redact.SafeString(i.Reason))
+		w.Printf("%s", i.Annotations)
+		w.Printf("%s; ", levelInfos(i.Input))
+		w.Printf("OverlappingRatio: Single %.2f, Multi %.2f", i.SingleLevelOverlappingRatio, i.MultiLevelOverlappingRatio)
 		return
 	}
 	outputSize := tablesTotalSize(i.Output.Tables)
 	w.Printf("[JOB %d] compacted(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
+	w.Printf("%s", i.Annotations)
 	w.Print(levelInfos(i.Input))
 	w.Printf(" -> L%d [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
 		redact.Safe(i.Output.Level),
 		redact.Safe(formatFileNums(i.Output.Tables)),
-		redact.Safe(humanize.Uint64(outputSize)),
+		redact.Safe(humanize.Bytes.Uint64(outputSize)),
 		redact.Safe(i.Duration.Seconds()),
 		redact.Safe(i.TotalDuration.Seconds()),
-		redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
+		redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
 }
 
 type levelInfos []LevelInfo
@@ -129,6 +159,9 @@ type FlushInfo struct {
 	Reason string
 	// Input contains the count of input memtables that were flushed.
 	Input int
+	// InputBytes contains the total in-memory size of the memtable(s) that were
+	// flushed. This size includes skiplist indexing data structures.
+	InputBytes uint64
 	// Output contains the ouptut table generated by the flush. The output info
 	// is empty for the flush begin event.
 	Output []TableInfo
@@ -168,7 +201,7 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		if !i.Ingest {
 			w.Printf("flushing %d memtable", redact.Safe(i.Input))
 			w.SafeString(plural)
-			w.Printf(" to L0")
+			w.Printf(" (%s) to L0", redact.Safe(humanize.Bytes.Uint64(i.InputBytes)))
 		} else {
 			w.Printf("flushing %d ingested table%s", redact.Safe(i.Input), plural)
 		}
@@ -180,13 +213,14 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		if invariants.Enabled && len(i.IngestLevels) > 0 {
 			panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) == 0"))
 		}
-		w.Printf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
+		w.Printf("[JOB %d] flushed %d memtable%s (%s) to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
 			redact.Safe(i.JobID), redact.Safe(i.Input), plural,
+			redact.Safe(humanize.Bytes.Uint64(i.InputBytes)),
 			redact.Safe(formatFileNums(i.Output)),
-			redact.Safe(humanize.Uint64(outputSize)),
+			redact.Safe(humanize.Bytes.Uint64(outputSize)),
 			redact.Safe(i.Duration.Seconds()),
 			redact.Safe(i.TotalDuration.Seconds()),
-			redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
+			redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
 	} else {
 		if invariants.Enabled && len(i.IngestLevels) == 0 {
 			panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) > 0"))
@@ -198,12 +232,12 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 			if j > 0 {
 				w.Printf(" +")
 			}
-			w.Printf(" L%d:%s (%s)", level, redact.Safe(file.FileNum), humanize.IEC.Uint64(file.Size))
+			w.Printf(" L%d:%s (%s)", level, file.FileNum, humanize.Bytes.Uint64(file.Size))
 		}
 		w.Printf(" in %.1fs (%.1fs total), output rate %s/s",
 			redact.Safe(i.Duration.Seconds()),
 			redact.Safe(i.TotalDuration.Seconds()),
-			redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
+			redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
 	}
 }
 
@@ -227,7 +261,7 @@ func (i ManifestCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		w.Printf("[JOB %d] MANIFEST create error: %s", redact.Safe(i.JobID), i.Err)
 		return
 	}
-	w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+	w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), i.FileNum)
 }
 
 // ManifestDeleteInfo contains the info for a Manifest deletion event.
@@ -249,7 +283,7 @@ func (i ManifestDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		w.Printf("[JOB %d] MANIFEST delete error: %s", redact.Safe(i.JobID), i.Err)
 		return
 	}
-	w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+	w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), i.FileNum)
 }
 
 // TableCreateInfo contains the info for a table creation event.
@@ -269,7 +303,7 @@ func (i TableCreateInfo) String() string {
 // SafeFormat implements redact.SafeFormatter.
 func (i TableCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 	w.Printf("[JOB %d] %s: sstable created %s",
-		redact.Safe(i.JobID), redact.Safe(i.Reason), redact.Safe(i.FileNum))
+		redact.Safe(i.JobID), redact.Safe(i.Reason), i.FileNum)
 }
 
 // TableDeleteInfo contains the info for a table deletion event.
@@ -288,10 +322,10 @@ func (i TableDeleteInfo) String() string {
 func (i TableDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 	if i.Err != nil {
 		w.Printf("[JOB %d] sstable delete error %s: %s",
-			redact.Safe(i.JobID), redact.Safe(i.FileNum), i.Err)
+			redact.Safe(i.JobID), i.FileNum, i.Err)
 		return
 	}
-	w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+	w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), i.FileNum)
 }
 
 // TableIngestInfo contains the info for a table ingestion event.
@@ -337,8 +371,8 @@ func (i TableIngestInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		if !i.flushable {
 			levelStr = fmt.Sprintf("L%d:", t.Level)
 		}
-		w.Printf(" %s%s (%s)", redact.Safe(levelStr), redact.Safe(t.FileNum),
+		w.Printf(" %s%s (%s)", redact.Safe(levelStr), t.FileNum,
-			redact.Safe(humanize.Uint64(t.Size)))
+			redact.Safe(humanize.Bytes.Uint64(t.Size)))
 	}
 }
 
@@ -399,12 +433,12 @@ func (i WALCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 	}
 
 	if i.RecycledFileNum == 0 {
-		w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+		w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), i.FileNum)
 		return
 	}
 
 	w.Printf("[JOB %d] WAL created %s (recycled %s)",
-		redact.Safe(i.JobID), redact.Safe(i.FileNum), redact.Safe(i.RecycledFileNum))
+		redact.Safe(i.JobID), i.FileNum, i.RecycledFileNum)
 }
 
 // WALDeleteInfo contains the info for a WAL deletion event.
@@ -426,7 +460,7 @@ func (i WALDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		w.Printf("[JOB %d] WAL delete error: %s", redact.Safe(i.JobID), i.Err)
 		return
 	}
-	w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+	w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), i.FileNum)
 }
 
 // WriteStallBeginInfo contains the info for a write stall begin event.

vendor/github.com/cockroachdb/pebble/external_iterator.go

@@ -311,7 +311,7 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) {
 					base.InternalKeySeqNumMax,
 					it.opts.LowerBound, it.opts.UpperBound,
 					&it.hasPrefix, &it.prefixOrFullSeekKey,
-					true /* onlySets */, &it.rangeKey.internal,
+					false /* internalKeys */, &it.rangeKey.internal,
 				)
 				for i := range rangeKeyIters {
 					it.rangeKey.iterConfig.AddLevel(rangeKeyIters[i])
@@ -319,8 +319,12 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) {
 			}
 		}
 		if it.rangeKey != nil {
-			it.rangeKey.iiter.Init(&it.comparer, it.iter, it.rangeKey.rangeKeyIter, &it.rangeKeyMasking,
+			it.rangeKey.iiter.Init(&it.comparer, it.iter, it.rangeKey.rangeKeyIter,
-				it.opts.LowerBound, it.opts.UpperBound)
+				keyspan.InterleavingIterOpts{
+					Mask:       &it.rangeKeyMasking,
+					LowerBound: it.opts.LowerBound,
+					UpperBound: it.opts.UpperBound,
+				})
 			it.iter = &it.rangeKey.iiter
 		}
 	}

vendor/github.com/cockroachdb/pebble/flushable.go

@@ -60,7 +60,7 @@ type flushableEntry struct {
 	// flushable is a memTable, when the reader refs drops to zero, the writer
 	// refs will already be zero because the memtable will have been flushed and
 	// that only occurs once the writer refs drops to zero.
-	readerRefs int32
+	readerRefs atomic.Int32
 	// Closure to invoke to release memory accounting.
 	releaseMemAccounting func()
 	// unrefFiles, if not nil, should be invoked to decrease the ref count of
@@ -73,7 +73,7 @@ type flushableEntry struct {
 }
 
 func (e *flushableEntry) readerRef() {
-	switch v := atomic.AddInt32(&e.readerRefs, 1); {
+	switch v := e.readerRefs.Add(1); {
 	case v <= 1:
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
 	}
@@ -92,7 +92,7 @@ func (e *flushableEntry) readerUnrefLocked(deleteFiles bool) {
 func (e *flushableEntry) readerUnrefHelper(
 	deleteFiles bool, deleteFn func(obsolete []*fileBacking),
 ) {
-	switch v := atomic.AddInt32(&e.readerRefs, -1); {
+	switch v := e.readerRefs.Add(-1); {
 	case v < 0:
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
 	case v == 0:
@@ -117,8 +117,7 @@ type flushableList []*flushableEntry
 // ingesting sstables which are added to the flushable list.
 type ingestedFlushable struct {
 	files            []physicalMeta
-	cmp              Compare
+	comparer         *Comparer
-	split            Split
 	newIters         tableNewIters
 	newRangeKeyIters keyspan.TableNewSpanIter
 
@@ -131,8 +130,7 @@ type ingestedFlushable struct {
 
 func newIngestedFlushable(
 	files []*fileMetadata,
-	cmp Compare,
+	comparer *Comparer,
-	split Split,
 	newIters tableNewIters,
 	newRangeKeyIters keyspan.TableNewSpanIter,
 ) *ingestedFlushable {
@@ -147,12 +145,11 @@ func newIngestedFlushable(
 
 	ret := &ingestedFlushable{
 		files:            physicalFiles,
-		cmp:              cmp,
+		comparer:         comparer,
-		split:            split,
 		newIters:         newIters,
 		newRangeKeyIters: newRangeKeyIters,
 		// slice is immutable and can be set once and used many times.
-		slice:        manifest.NewLevelSliceKeySorted(cmp, files),
+		slice:        manifest.NewLevelSliceKeySorted(comparer.Compare, files),
 		hasRangeKeys: hasRangeKeys,
 	}
 
@@ -162,6 +159,7 @@ func newIngestedFlushable(
 // TODO(sumeer): ingestedFlushable iters also need to plumb context for
 // tracing.
 
+// newIter is part of the flushable interface.
 func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator {
 	var opts IterOptions
 	if o != nil {
@@ -172,10 +170,11 @@ func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator {
 	// aren't truly levels in the lsm. Right now, the encoding only supports
 	// L0 sublevels, and the rest of the levels in the lsm.
 	return newLevelIter(
-		opts, s.cmp, s.split, s.newIters, s.slice.Iter(), manifest.Level(0), nil,
+		opts, s.comparer, s.newIters, s.slice.Iter(), manifest.Level(0), internalIterOpts{},
 	)
 }
 
+// newFlushIter is part of the flushable interface.
 func (s *ingestedFlushable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
 	// newFlushIter is only used for writing memtables to disk as sstables.
 	// Since ingested sstables are already present on disk, they don't need to
@@ -199,41 +198,47 @@ func (s *ingestedFlushable) constructRangeDelIter(
 	return rangeDelIter, nil
 }
 
+// newRangeDelIter is part of the flushable interface.
 // TODO(bananabrick): Using a level iter instead of a keyspan level iter to
 // surface range deletes is more efficient.
 func (s *ingestedFlushable) newRangeDelIter(_ *IterOptions) keyspan.FragmentIterator {
 	return keyspan.NewLevelIter(
-		keyspan.SpanIterOptions{}, s.cmp,
+		keyspan.SpanIterOptions{}, s.comparer.Compare,
 		s.constructRangeDelIter, s.slice.Iter(), manifest.Level(0),
 		manifest.KeyTypePoint,
 	)
 }
 
+// newRangeKeyIter is part of the flushable interface.
 func (s *ingestedFlushable) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator {
 	if !s.containsRangeKeys() {
 		return nil
 	}
 
 	return keyspan.NewLevelIter(
-		keyspan.SpanIterOptions{}, s.cmp, s.newRangeKeyIters,
+		keyspan.SpanIterOptions{}, s.comparer.Compare, s.newRangeKeyIters,
 		s.slice.Iter(), manifest.Level(0), manifest.KeyTypeRange,
 	)
 }
 
+// containsRangeKeys is part of the flushable interface.
 func (s *ingestedFlushable) containsRangeKeys() bool {
 	return s.hasRangeKeys
 }
 
+// inuseBytes is part of the flushable interface.
 func (s *ingestedFlushable) inuseBytes() uint64 {
 	// inuseBytes is only used when memtables are flushed to disk as sstables.
 	panic("pebble: not implemented")
 }
 
+// totalBytes is part of the flushable interface.
 func (s *ingestedFlushable) totalBytes() uint64 {
 	// We don't allocate additional bytes for the ingestedFlushable.
 	return 0
 }
 
+// readyForFlush is part of the flushable interface.
 func (s *ingestedFlushable) readyForFlush() bool {
 	// ingestedFlushable should always be ready to flush. However, note that
 	// memtables before the ingested sstables in the memtable queue must be

vendor/github.com/cockroachdb/pebble/format_major_version.go

@@ -28,10 +28,13 @@ import (
 // FormatVersion that the default corresponds to may change with time.
 type FormatMajorVersion uint64
 
+// SafeValue implements redact.SafeValue.
+func (v FormatMajorVersion) SafeValue() {}
+
 // String implements fmt.Stringer.
 func (v FormatMajorVersion) String() string {
-	// NB: This must not change. It's used as the value for the the
+	// NB: This must not change. It's used as the value for the on-disk
-	// on-disk version marker file.
+	// version marker file.
 	//
 	// Specifically, this value must always parse as a base 10 integer
 	// that fits in a uint64. We format it as zero-padded, 3-digit
@@ -110,12 +113,12 @@ const (
 
 	// 23.1 versions.
 
-	// FormatUnusedPrePebblev1MarkedCompacted is an unused format major version.
+	// formatUnusedPrePebblev1MarkedCompacted is an unused format major version.
 	// This format major version was originally intended to ship in the 23.1
 	// release. It was later decided that this should be deferred until a
 	// subsequent release. The original ordering is preserved so as not to
 	// introduce breaking changes in Cockroach.
-	FormatUnusedPrePebblev1MarkedCompacted
+	formatUnusedPrePebblev1MarkedCompacted
 
 	// FormatSSTableValueBlocks is a format major version that adds support for
 	// storing values in value blocks in the sstable. Value block support is not
@@ -151,20 +154,20 @@ const (
 	// compactions for files marked for compaction are complete.
 	FormatPrePebblev1MarkedCompacted
 
-	// ExperimentalFormatDeleteSizedAndObsolete is a format major version that adds support
+	// FormatDeleteSizedAndObsolete is a format major version that adds support
 	// for deletion tombstones that encode the size of the value they're
 	// expected to delete. This format major version is required before the
 	// associated key kind may be committed through batch applications or
 	// ingests. It also adds support for keys that are marked obsolete (see
 	// sstable/format.go for details).
-	ExperimentalFormatDeleteSizedAndObsolete
+	FormatDeleteSizedAndObsolete
 
-	// ExperimentalFormatVirtualSSTables is a format major version that adds support for
+	// FormatVirtualSSTables is a format major version that adds support for
 	// virtual sstables that can reference a sub-range of keys in an underlying
 	// physical sstable. This information is persisted through new,
 	// backward-incompatible fields in the Manifest, and therefore requires
 	// a format major version.
-	ExperimentalFormatVirtualSSTables
+	FormatVirtualSSTables
 
 	// internalFormatNewest holds the newest format major version, including
 	// experimental ones excluded from the exported FormatNewest constant until
@@ -172,8 +175,7 @@ const (
 	internalFormatNewest FormatMajorVersion = iota - 1
 
 	// FormatNewest always contains the most recent format major version.
-	// TODO(jackson): restore FormatNewest to internalFormatNewest.
+	FormatNewest FormatMajorVersion = internalFormatNewest
-	FormatNewest FormatMajorVersion = FormatPrePebblev1MarkedCompacted
 )
 
 // MaxTableFormat returns the maximum sstable.TableFormat that can be used at
@@ -187,11 +189,11 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
 		FormatSplitUserKeysMarkedCompacted:
 		return sstable.TableFormatPebblev1
 	case FormatRangeKeys, FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
-		FormatUnusedPrePebblev1MarkedCompacted:
+		formatUnusedPrePebblev1MarkedCompacted:
 		return sstable.TableFormatPebblev2
 	case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
 		return sstable.TableFormatPebblev3
-	case ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables:
+	case FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
 		return sstable.TableFormatPebblev4
 	default:
 		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
@@ -208,15 +210,27 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
 		FormatRangeKeys:
 		return sstable.TableFormatLevelDB
 	case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
-		FormatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
+		formatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
 		FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
-		ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables:
+		FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
 		return sstable.TableFormatPebblev1
 	default:
 		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
 	}
 }
 
+// orderingInvariants returns an enum encoding the set of invariants that must
+// hold within the receiver format major version. Invariants only get stricter
+// as the format major version advances, so it is okay to retrieve the
+// invariants from the current format major version and by the time the
+// invariants are enforced, the format major version has advanced.
+func (v FormatMajorVersion) orderingInvariants() manifest.OrderingInvariants {
+	if v < FormatSplitUserKeysMarkedCompacted {
+		return manifest.AllowSplitUserKeys
+	}
+	return manifest.ProhibitSplitUserKeys
+}
+
 // formatMajorVersionMigrations defines the migrations from one format
 // major version to the next. Each migration is defined as a closure
 // which will be invoked on the database before the new format major
@@ -324,9 +338,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 		}
 		return d.finalizeFormatVersUpgrade(FormatPrePebblev1Marked)
 	},
-	FormatUnusedPrePebblev1MarkedCompacted: func(d *DB) error {
+	formatUnusedPrePebblev1MarkedCompacted: func(d *DB) error {
 		// Intentional no-op.
-		return d.finalizeFormatVersUpgrade(FormatUnusedPrePebblev1MarkedCompacted)
+		return d.finalizeFormatVersUpgrade(formatUnusedPrePebblev1MarkedCompacted)
 	},
 	FormatSSTableValueBlocks: func(d *DB) error {
 		return d.finalizeFormatVersUpgrade(FormatSSTableValueBlocks)
@@ -343,11 +357,11 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 		}
 		return d.finalizeFormatVersUpgrade(FormatPrePebblev1MarkedCompacted)
 	},
-	ExperimentalFormatDeleteSizedAndObsolete: func(d *DB) error {
+	FormatDeleteSizedAndObsolete: func(d *DB) error {
-		return d.finalizeFormatVersUpgrade(ExperimentalFormatDeleteSizedAndObsolete)
+		return d.finalizeFormatVersUpgrade(FormatDeleteSizedAndObsolete)
 	},
-	ExperimentalFormatVirtualSSTables: func(d *DB) error {
+	FormatVirtualSSTables: func(d *DB) error {
-		return d.finalizeFormatVersUpgrade(ExperimentalFormatVirtualSSTables)
+		return d.finalizeFormatVersUpgrade(FormatVirtualSSTables)
 	},
 }
 
@@ -382,9 +396,7 @@ func lookupFormatMajorVersion(
 // provided in Options when the database was opened if the existing
 // database was written with a higher format version.
 func (d *DB) FormatMajorVersion() FormatMajorVersion {
-	d.mu.Lock()
+	return FormatMajorVersion(d.mu.formatVers.vers.Load())
-	defer d.mu.Unlock()
-	return d.mu.formatVers.vers
 }
 
 // RatchetFormatMajorVersion ratchets the opened database's format major
@@ -411,9 +423,9 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro
 		// Guard against accidentally forgetting to update internalFormatNewest.
 		return errors.Errorf("pebble: unknown format version %d", formatVers)
 	}
-	if d.mu.formatVers.vers > formatVers {
+	if currentVers := d.FormatMajorVersion(); currentVers > formatVers {
 		return errors.Newf("pebble: database already at format major version %d; cannot reduce to %d",
-			d.mu.formatVers.vers, formatVers)
+			currentVers, formatVers)
 	}
 	if d.mu.formatVers.ratcheting {
 		return errors.Newf("pebble: database format major version upgrade is in-progress")
@@ -421,7 +433,7 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro
 	d.mu.formatVers.ratcheting = true
 	defer func() { d.mu.formatVers.ratcheting = false }()
 
-	for nextVers := d.mu.formatVers.vers + 1; nextVers <= formatVers; nextVers++ {
+	for nextVers := d.FormatMajorVersion() + 1; nextVers <= formatVers; nextVers++ {
 		if err := formatMajorVersionMigrations[nextVers](d); err != nil {
 			return errors.Wrapf(err, "migrating to version %d", nextVers)
 		}
@@ -432,7 +444,7 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro
 		// update in-memory state (without ever dropping locks) after
 		// the upgrade is finalized. Here we assert that the upgrade
 		// did occur.
-		if d.mu.formatVers.vers != nextVers {
+		if d.FormatMajorVersion() != nextVers {
 			d.opts.Logger.Fatalf("pebble: successful migration to format version %d never finalized the upgrade", nextVers)
 		}
 	}
@@ -451,7 +463,7 @@ func (d *DB) finalizeFormatVersUpgrade(formatVers FormatMajorVersion) error {
 	if err := d.mu.formatVers.marker.Move(formatVers.String()); err != nil {
 		return err
 	}
-	d.mu.formatVers.vers = formatVers
+	d.mu.formatVers.vers.Store(uint64(formatVers))
 	d.opts.EventListener.FormatUpgrade(formatVers)
 	return nil
 }

vendor/github.com/cockroachdb/pebble/get_iter.go

@@ -19,8 +19,7 @@ import (
 // lazily.
 type getIter struct {
 	logger       Logger
-	cmp          Compare
+	comparer     *Comparer
-	equal        Equal
 	newIters     tableNewIters
 	snapshot     uint64
 	key          []byte
@@ -83,7 +82,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 			// key. Every call to levelIter.Next() potentially switches to a new
 			// table and thus reinitializes rangeDelIter.
 			if g.rangeDelIter != nil {
-				g.tombstone = keyspan.Get(g.cmp, g.rangeDelIter, g.key)
+				g.tombstone = keyspan.Get(g.comparer.Compare, g.rangeDelIter, g.key)
 				if g.err = g.rangeDelIter.Close(); g.err != nil {
 					return nil, base.LazyValue{}
 				}
@@ -101,7 +100,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 					g.iter = nil
 					return nil, base.LazyValue{}
 				}
-				if g.equal(g.key, key.UserKey) {
+				if g.comparer.Equal(g.key, key.UserKey) {
 					if !key.Visible(g.snapshot, base.InternalKeySeqNumMax) {
 						g.iterKey, g.iterValue = g.iter.Next()
 						continue
@@ -159,11 +158,24 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 				files := g.l0[n-1].Iter()
 				g.l0 = g.l0[:n-1]
 				iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
-				g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters,
+				g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters,
 					files, manifest.L0Sublevel(n), internalIterOpts{})
 				g.levelIter.initRangeDel(&g.rangeDelIter)
+				bc := levelIterBoundaryContext{}
+				g.levelIter.initBoundaryContext(&bc)
 				g.iter = &g.levelIter
-				g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone)
+
+				// Compute the key prefix for bloom filtering if split function is
+				// specified, or use the user key as default.
+				prefix := g.key
+				if g.comparer.Split != nil {
+					prefix = g.key[:g.comparer.Split(g.key)]
+				}
+				g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
+				if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {
+					g.iterKey = nil
+					g.iterValue = base.LazyValue{}
+				}
 				continue
 			}
 			g.level++
@@ -178,12 +190,25 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 		}
 
 		iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
-		g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters,
+		g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters,
 			g.version.Levels[g.level].Iter(), manifest.Level(g.level), internalIterOpts{})
 		g.levelIter.initRangeDel(&g.rangeDelIter)
+		bc := levelIterBoundaryContext{}
+		g.levelIter.initBoundaryContext(&bc)
 		g.level++
 		g.iter = &g.levelIter
-		g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone)
+
+		// Compute the key prefix for bloom filtering if split function is
+		// specified, or use the user key as default.
+		prefix := g.key
+		if g.comparer.Split != nil {
+			prefix = g.key[:g.comparer.Split(g.key)]
+		}
+		g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
+		if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {
+			g.iterKey = nil
+			g.iterValue = base.LazyValue{}
+		}
 	}
 }
 

vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go

@@ -18,22 +18,21 @@
 package arenaskl
 
 import (
-	"math"
 	"sync/atomic"
 	"unsafe"
 
 	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/constants"
+	"github.com/cockroachdb/pebble/internal/invariants"
 )
 
 // Arena is lock-free.
 type Arena struct {
-	n   uint64
+	n   atomic.Uint64
 	buf []byte
 }
 
-const (
+const nodeAlignment = 4
-	align4 = 3
-)
 
 var (
 	// ErrArenaFull indicates that the arena is full and cannot perform any more
@@ -44,20 +43,28 @@ var (
 // NewArena allocates a new arena using the specified buffer as the backing
 // store.
 func NewArena(buf []byte) *Arena {
-	// Don't store data at position 0 in order to reserve offset=0 as a kind
+	if len(buf) > constants.MaxUint32OrInt {
-	// of nil pointer.
+		if invariants.Enabled {
-	return &Arena{
+			panic(errors.AssertionFailedf("attempting to create arena of size %d", len(buf)))
-		n:   1,
+		}
+		buf = buf[:constants.MaxUint32OrInt]
+	}
+	a := &Arena{
 		buf: buf,
 	}
+	// We don't store data at position 0 in order to reserve offset=0 as a kind of
+	// nil pointer.
+	a.n.Store(1)
+	return a
 }
 
 // Size returns the number of bytes allocated by the arena.
 func (a *Arena) Size() uint32 {
-	s := atomic.LoadUint64(&a.n)
+	s := a.n.Load()
-	if s > math.MaxUint32 {
+	if s > constants.MaxUint32OrInt {
-		// Saturate at MaxUint32.
+		// The last failed allocation can push the size higher than len(a.buf).
-		return math.MaxUint32
+		// Saturate at the maximum representable offset.
+		return constants.MaxUint32OrInt
 	}
 	return uint32(s)
 }
@@ -67,24 +74,33 @@ func (a *Arena) Capacity() uint32 {
 	return uint32(len(a.buf))
 }
 
-func (a *Arena) alloc(size, align, overflow uint32) (uint32, uint32, error) {
+// alloc allocates a buffer of the given size and with the given alignment
+// (which must be a power of 2).
+//
+// If overflow is not 0, it also ensures that many bytes after the buffer are
+// inside the arena (this is used for structures that are larger than the
+// requested size but don't use those extra bytes).
+func (a *Arena) alloc(size, alignment, overflow uint32) (uint32, uint32, error) {
+	if invariants.Enabled && (alignment&(alignment-1)) != 0 {
+		panic(errors.AssertionFailedf("invalid alignment %d", alignment))
+	}
 	// Verify that the arena isn't already full.
-	origSize := atomic.LoadUint64(&a.n)
+	origSize := a.n.Load()
 	if int(origSize) > len(a.buf) {
 		return 0, 0, ErrArenaFull
 	}
 
 	// Pad the allocation with enough bytes to ensure the requested alignment.
-	padded := uint32(size) + align
+	padded := uint64(size) + uint64(alignment) - 1
 
-	newSize := atomic.AddUint64(&a.n, uint64(padded))
+	newSize := a.n.Add(padded)
-	if int(newSize)+int(overflow) > len(a.buf) {
+	if newSize+uint64(overflow) > uint64(len(a.buf)) {
 		return 0, 0, ErrArenaFull
 	}
 
 	// Return the aligned offset.
-	offset := (uint32(newSize) - padded + align) & ^align
+	offset := (uint32(newSize) - size) & ^(alignment - 1)
-	return offset, padded, nil
+	return offset, uint32(padded), nil
 }
 
 func (a *Arena) getBytes(offset uint32, size uint32) []byte {

vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go

@@ -18,7 +18,6 @@
 package arenaskl
 
 import (
-	"encoding/binary"
 	"sync"
 
 	"github.com/cockroachdb/pebble/internal/base"
@@ -240,17 +239,8 @@ func (it *Iterator) SetBounds(lower, upper []byte) {
 }
 
 func (it *Iterator) decodeKey() {
-	b := it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
+	it.key.UserKey = it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize)
-	// This is a manual inline of base.DecodeInternalKey, because the Go compiler
+	it.key.Trailer = it.nd.keyTrailer
-	// seems to refuse to automatically inline it currently.
-	l := len(b) - 8
-	if l >= 0 {
-		it.key.Trailer = binary.LittleEndian.Uint64(b[l:])
-		it.key.UserKey = b[:l:l]
-	} else {
-		it.key.Trailer = uint64(base.InternalKeyKindInvalid)
-		it.key.UserKey = nil
-	}
 }
 
 func (it *Iterator) seekForBaseSplice(key []byte) (prev, next *node, found bool) {

vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go

@@ -29,25 +29,27 @@ import (
 // is used here. If a key/value overflows a uint32, it should not be added to
 // the skiplist.
 func MaxNodeSize(keySize, valueSize uint32) uint64 {
-	return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + align4
+	const maxPadding = nodeAlignment - 1
+	return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + maxPadding
 }
 
 type links struct {
-	nextOffset uint32
+	nextOffset atomic.Uint32
-	prevOffset uint32
+	prevOffset atomic.Uint32
 }
 
 func (l *links) init(prevOffset, nextOffset uint32) {
-	l.nextOffset = nextOffset
+	l.nextOffset.Store(nextOffset)
-	l.prevOffset = prevOffset
+	l.prevOffset.Store(prevOffset)
 }
 
 type node struct {
 	// Immutable fields, so no need to lock to access key.
-	keyOffset uint32
+	keyOffset  uint32
-	keySize   uint32
+	keySize    uint32
-	valueSize uint32
+	keyTrailer uint64
-	allocSize uint32
+	valueSize  uint32
+	allocSize  uint32
 
 	// Most nodes do not need to use the full height of the tower, since the
 	// probability of each successive level decreases exponentially. Because
@@ -65,7 +67,7 @@ func newNode(
 	if height < 1 || height > maxHeight {
 		panic("height cannot be less than one or greater than the max height")
 	}
-	keySize := key.Size()
+	keySize := len(key.UserKey)
 	if int64(keySize) > math.MaxUint32 {
 		panic("key is too large")
 	}
@@ -81,8 +83,8 @@ func newNode(
 	if err != nil {
 		return
 	}
-
+	nd.keyTrailer = key.Trailer
-	key.Encode(nd.getKeyBytes(arena))
+	copy(nd.getKeyBytes(arena), key.UserKey)
 	copy(nd.getValue(arena), value)
 	return
 }
@@ -93,7 +95,7 @@ func newRawNode(arena *Arena, height uint32, keySize, valueSize uint32) (nd *nod
 	unusedSize := uint32((maxHeight - int(height)) * linksSize)
 	nodeSize := uint32(maxNodeSize) - unusedSize
 
-	nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, align4, unusedSize)
+	nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, nodeAlignment, unusedSize)
 	if err != nil {
 		return
 	}
@@ -115,17 +117,17 @@ func (n *node) getValue(arena *Arena) []byte {
 }
 
 func (n *node) nextOffset(h int) uint32 {
-	return atomic.LoadUint32(&n.tower[h].nextOffset)
+	return n.tower[h].nextOffset.Load()
 }
 
 func (n *node) prevOffset(h int) uint32 {
-	return atomic.LoadUint32(&n.tower[h].prevOffset)
+	return n.tower[h].prevOffset.Load()
 }
 
 func (n *node) casNextOffset(h int, old, val uint32) bool {
-	return atomic.CompareAndSwapUint32(&n.tower[h].nextOffset, old, val)
+	return n.tower[h].nextOffset.CompareAndSwap(old, val)
 }
 
 func (n *node) casPrevOffset(h int, old, val uint32) bool {
-	return atomic.CompareAndSwapUint32(&n.tower[h].prevOffset, old, val)
+	return n.tower[h].prevOffset.CompareAndSwap(old, val)
 }

vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go

@@ -44,7 +44,6 @@ Key differences:
 package arenaskl // import "github.com/cockroachdb/pebble/internal/arenaskl"
 
 import (
-	"encoding/binary"
 	"math"
 	"runtime"
 	"sync/atomic"
@@ -80,7 +79,7 @@ type Skiplist struct {
 	cmp    base.Compare
 	head   *node
 	tail   *node
-	height uint32 // Current height. 1 <= height <= maxHeight. CAS.
+	height atomic.Uint32 // Current height. 1 <= height <= maxHeight. CAS.
 
 	// If set to true by tests, then extra delays are added to make it easier to
 	// detect unusual race conditions.
@@ -140,22 +139,22 @@ func (s *Skiplist) Reset(arena *Arena, cmp base.Compare) {
 	headOffset := arena.getPointerOffset(unsafe.Pointer(head))
 	tailOffset := arena.getPointerOffset(unsafe.Pointer(tail))
 	for i := 0; i < maxHeight; i++ {
-		head.tower[i].nextOffset = tailOffset
+		head.tower[i].nextOffset.Store(tailOffset)
-		tail.tower[i].prevOffset = headOffset
+		tail.tower[i].prevOffset.Store(headOffset)
 	}
 
 	*s = Skiplist{
-		arena:  arena,
+		arena: arena,
-		cmp:    cmp,
+		cmp:   cmp,
-		head:   head,
+		head:  head,
-		tail:   tail,
+		tail:  tail,
-		height: 1,
 	}
+	s.height.Store(1)
 }
 
 // Height returns the height of the highest tower within any of the nodes that
 // have ever been allocated as part of this skiplist.
-func (s *Skiplist) Height() uint32 { return atomic.LoadUint32(&s.height) }
+func (s *Skiplist) Height() uint32 { return s.height.Load() }
 
 // Arena returns the arena backing this skiplist.
 func (s *Skiplist) Arena() *Arena { return s.arena }
@@ -327,7 +326,7 @@ func (s *Skiplist) newNode(
 	// Try to increase s.height via CAS.
 	listHeight := s.Height()
 	for height > listHeight {
-		if atomic.CompareAndSwapUint32(&s.height, listHeight, height) {
+		if s.height.CompareAndSwap(listHeight, height) {
 			// Successfully increased skiplist.height.
 			break
 		}
@@ -412,26 +411,19 @@ func (s *Skiplist) findSpliceForLevel(
 
 		offset, size := next.keyOffset, next.keySize
 		nextKey := s.arena.buf[offset : offset+size]
-		n := int32(size) - 8
+		cmp := s.cmp(key.UserKey, nextKey)
-		cmp := s.cmp(key.UserKey, nextKey[:n])
 		if cmp < 0 {
 			// We are done for this level, since prev.key < key < next.key.
 			break
 		}
 		if cmp == 0 {
 			// User-key equality.
-			var nextTrailer uint64
+			if key.Trailer == next.keyTrailer {
-			if n >= 0 {
-				nextTrailer = binary.LittleEndian.Uint64(nextKey[n:])
-			} else {
-				nextTrailer = uint64(base.InternalKeyKindInvalid)
-			}
-			if key.Trailer == nextTrailer {
 				// Internal key equality.
 				found = true
 				break
 			}
-			if key.Trailer > nextTrailer {
+			if key.Trailer > next.keyTrailer {
 				// We are done for this level, since prev.key < key < next.key.
 				break
 			}
@@ -446,8 +438,7 @@ func (s *Skiplist) findSpliceForLevel(
 
 func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
 	ndKey := s.arena.buf[nd.keyOffset : nd.keyOffset+nd.keySize]
-	n := int32(nd.keySize) - 8
+	cmp := s.cmp(ndKey, key.UserKey)
-	cmp := s.cmp(ndKey[:n], key.UserKey)
 	if cmp < 0 {
 		return true
 	}
@@ -455,25 +446,19 @@ func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool {
 		return false
 	}
 	// User-key equality.
-	var ndTrailer uint64
+	if key.Trailer == nd.keyTrailer {
-	if n >= 0 {
-		ndTrailer = binary.LittleEndian.Uint64(ndKey[n:])
-	} else {
-		ndTrailer = uint64(base.InternalKeyKindInvalid)
-	}
-	if key.Trailer == ndTrailer {
 		// Internal key equality.
 		return false
 	}
-	return key.Trailer < ndTrailer
+	return key.Trailer < nd.keyTrailer
 }
 
 func (s *Skiplist) getNext(nd *node, h int) *node {
-	offset := atomic.LoadUint32(&nd.tower[h].nextOffset)
+	offset := nd.tower[h].nextOffset.Load()
 	return (*node)(s.arena.getPointer(offset))
 }
 
 func (s *Skiplist) getPrev(nd *node, h int) *node {
-	offset := atomic.LoadUint32(&nd.tower[h].prevOffset)
+	offset := nd.tower[h].prevOffset.Load()
 	return (*node)(s.arena.getPointer(offset))
 }

vendor/github.com/cockroachdb/pebble/internal/base/filenames.go

@@ -20,6 +20,11 @@ type FileNum uint64
 // String returns a string representation of the file number.
 func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
 
+// SafeFormat implements redact.SafeFormatter.
+func (fn FileNum) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Printf("%06d", redact.SafeUint(fn))
+}
+
 // DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
 // called if the caller can ensure that the FileNum belongs to a physical file
 // on disk. These could be manifests, log files, physical sstables on disk, the
@@ -40,6 +45,11 @@ type DiskFileNum struct {
 
 func (dfn DiskFileNum) String() string { return dfn.fn.String() }
 
+// SafeFormat implements redact.SafeFormatter.
+func (dfn DiskFileNum) SafeFormat(w redact.SafePrinter, verb rune) {
+	dfn.fn.SafeFormat(w, verb)
+}
+
 // FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
 func (dfn DiskFileNum) FileNum() FileNum {
 	return dfn.fn

vendor/github.com/cockroachdb/pebble/internal/base/internal.go

@@ -9,6 +9,17 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+
+	"github.com/cockroachdb/redact"
+)
+
+const (
+	// SeqNumZero is the zero sequence number, set by compactions if they can
+	// guarantee there are no keys underneath an internal key.
+	SeqNumZero = uint64(0)
+	// SeqNumStart is the first sequence number assigned to a key. Sequence
+	// numbers 1-9 are reserved for potential future use.
+	SeqNumStart = uint64(10)
 )
 
 // InternalKeyKind enumerates the kind of key: a deletion tombstone, a set
@@ -152,6 +163,11 @@ func (k InternalKeyKind) String() string {
 	return fmt.Sprintf("UNKNOWN:%d", k)
 }
 
+// SafeFormat implements redact.SafeFormatter.
+func (k InternalKeyKind) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Print(redact.SafeString(k.String()))
+}
+
 // InternalKey is a key used for the in-memory and on-disk partial DBs that
 // make up a pebble DB.
 //
@@ -365,6 +381,11 @@ func (k InternalKey) SeqNum() uint64 {
 	return k.Trailer >> 8
 }
 
+// SeqNumFromTrailer returns the sequence number component of a trailer.
+func SeqNumFromTrailer(t uint64) uint64 {
+	return t >> 8
+}
+
 // Visible returns true if the key is visible at the specified snapshot
 // sequence number.
 func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool {

vendor/github.com/cockroachdb/pebble/internal/base/iterator.go

@@ -7,6 +7,9 @@ package base
 import (
 	"fmt"
 	"time"
+
+	"github.com/cockroachdb/pebble/internal/humanize"
+	"github.com/cockroachdb/redact"
 )
 
 // InternalIterator iterates over a DB's key/value pairs in key order. Unlike
@@ -407,3 +410,35 @@ func (s *InternalIteratorStats) Merge(from InternalIteratorStats) {
 	s.SeparatedPointValue.ValueBytes += from.SeparatedPointValue.ValueBytes
 	s.SeparatedPointValue.ValueBytesFetched += from.SeparatedPointValue.ValueBytesFetched
 }
+
+func (s *InternalIteratorStats) String() string {
+	return redact.StringWithoutMarkers(s)
+}
+
+// SafeFormat implements the redact.SafeFormatter interface.
+func (s *InternalIteratorStats) SafeFormat(p redact.SafePrinter, verb rune) {
+	p.Printf("blocks: %s cached",
+		humanize.Bytes.Uint64(s.BlockBytesInCache),
+	)
+	if s.BlockBytes != s.BlockBytesInCache || s.BlockReadDuration != 0 {
+		p.Printf(", %s not cached (read time: %s)",
+			humanize.Bytes.Uint64(s.BlockBytes-s.BlockBytesInCache),
+			humanize.FormattedString(s.BlockReadDuration.String()),
+		)
+	}
+	p.Printf("; points: %s", humanize.Count.Uint64(s.PointCount))
+
+	if s.PointsCoveredByRangeTombstones != 0 {
+		p.Printf("(%s tombstoned)", humanize.Count.Uint64(s.PointsCoveredByRangeTombstones))
+	}
+	p.Printf(" (%s keys, %s values)",
+		humanize.Bytes.Uint64(s.KeyBytes),
+		humanize.Bytes.Uint64(s.ValueBytes),
+	)
+	if s.SeparatedPointValue.Count != 0 {
+		p.Printf("; separated: %s (%s, %s fetched)",
+			humanize.Count.Uint64(s.SeparatedPointValue.Count),
+			humanize.Bytes.Uint64(s.SeparatedPointValue.ValueBytes),
+			humanize.Bytes.Uint64(s.SeparatedPointValue.ValueBytesFetched))
+	}
+}

vendor/github.com/cockroachdb/pebble/internal/base/seqnums.go

@@ -1,56 +0,0 @@
-// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
-// of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package base
-
-import "fmt"
-
-// This file defines sequence numbers that are reserved for foreign keys i.e.
-// internal keys coming from other Pebble instances and existing in shared
-// storage, as those will "slot below" any internal keys added by our own Pebble
-// instance. Any keys created by this Pebble instance need to be greater than
-// all reserved sequence numbers (i.e. >= SeqNumStart).
-const (
-	// SeqNumZero is the zero sequence number, set by compactions if they can
-	// guarantee there are no keys underneath an internal key.
-	SeqNumZero = uint64(0)
-
-	// SeqNumL6 is the sequence number reserved for foreign keys in L6. This seqnum
-	// will be used to expose any range key sets as well as point keys in L6. Range
-	// deletes do not need to be exposed in L6.
-	SeqNumL6 = uint64(1)
-
-	// SeqNumL5 is the sequence number reserved for foreign keys in L5. This seqnum
-	// needs to be greater than SeqNumL6, so that range deletes in L5 can be
-	// exposed at this level and will correctly delete covering points in L6. Also
-	// range key unsets/dels will be exposed at this seqnum and will need to shadow
-	// overlapping range keys in L6.
-	//
-	// Note that we can use the same sequence number for all exposed keys in L5 as
-	// range dels do not delete points at the same seqnum, and range key
-	// unsets/deletes do not coalesce with range key sets at the same seqnum. Range
-	// key masking does not care about the sequence number of overlapping points
-	// (rather, it applies to points based on suffixes), so we can use this seqnum
-	// for all L5 keys exposed.
-	SeqNumL5 = uint64(2)
-
-	// Sequence numbers 3-9 are reserved for future use.
-
-	// SeqNumStart is the first sequence number assigned to a key written by
-	// ourselves.
-	SeqNumStart = uint64(10)
-)
-
-// SeqNumForLevel returns the appropriate reserved sequence number for keys in
-// foreign sstables at the specified level.
-func SeqNumForLevel(level int) uint64 {
-	switch level {
-	case 5:
-		return SeqNumL5
-	case 6:
-		return SeqNumL6
-	default:
-		panic(fmt.Sprintf("unexpected foreign sstable at level %d", level))
-	}
-}

vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go

@@ -63,14 +63,15 @@ import (
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/constants"
 	"golang.org/x/exp/rand"
 )
 
 const (
 	maxHeight    = 20
-	maxNodeSize  = int(unsafe.Sizeof(node{}))
+	maxNodeSize  = uint64(unsafe.Sizeof(node{}))
-	linksSize    = int(unsafe.Sizeof(links{}))
+	linksSize    = uint64(unsafe.Sizeof(links{}))
-	maxNodesSize = math.MaxUint32
+	maxNodesSize = constants.MaxUint32OrInt
 )
 
 var (
@@ -281,7 +282,7 @@ func (s *Skiplist) newNode(
 		panic("height cannot be less than one or greater than the max height")
 	}
 
-	unusedSize := (maxHeight - int(height)) * linksSize
+	unusedSize := uint64(maxHeight-int(height)) * linksSize
 	nodeOffset, err := s.alloc(uint32(maxNodeSize - unusedSize))
 	if err != nil {
 		return 0, err
@@ -296,13 +297,13 @@ func (s *Skiplist) newNode(
 }
 
 func (s *Skiplist) alloc(size uint32) (uint32, error) {
-	offset := len(s.nodes)
+	offset := uint64(len(s.nodes))
 
 	// We only have a need for memory up to offset + size, but we never want
 	// to allocate a node whose tail points into unallocated memory.
 	minAllocSize := offset + maxNodeSize
-	if cap(s.nodes) < minAllocSize {
+	if uint64(cap(s.nodes)) < minAllocSize {
-		allocSize := cap(s.nodes) * 2
+		allocSize := uint64(cap(s.nodes)) * 2
 		if allocSize < minAllocSize {
 			allocSize = minAllocSize
 		}

vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go

@@ -71,14 +71,12 @@ func (h Handle) Get() []byte {
 
 // Release releases the reference to the cache entry.
 func (h Handle) Release() {
-	if h.value != nil {
+	h.value.release()
-		h.value.release()
-	}
 }
 
 type shard struct {
-	hits   int64
+	hits   atomic.Int64
-	misses int64
+	misses atomic.Int64
 
 	mu sync.RWMutex
 
@@ -119,15 +117,15 @@ func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle {
 	if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil {
 		value = e.acquireValue()
 		if value != nil {
-			atomic.StoreInt32(&e.referenced, 1)
+			e.referenced.Store(true)
 		}
 	}
 	c.mu.RUnlock()
 	if value == nil {
-		atomic.AddInt64(&c.misses, 1)
+		c.misses.Add(1)
 		return Handle{}
 	}
-	atomic.AddInt64(&c.hits, 1)
+	c.hits.Add(1)
 	return Handle{value: value}
 }
 
@@ -160,7 +158,7 @@ func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *V
 	case e.peekValue() != nil:
 		// cache entry was a hot or cold page
 		e.setValue(value)
-		atomic.StoreInt32(&e.referenced, 1)
+		e.referenced.Store(true)
 		delta := int64(len(value.buf)) - e.size
 		e.size = int64(len(value.buf))
 		if e.ptype == etHot {
@@ -176,7 +174,7 @@ func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *V
 		// cache entry was a test page
 		c.sizeTest -= e.size
 		c.countTest--
-		c.metaDel(e)
+		c.metaDel(e).release()
 		c.metaCheck(e)
 
 		e.size = int64(len(value.buf))
@@ -185,7 +183,7 @@ func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *V
 			c.coldTarget = c.targetSize()
 		}
 
-		atomic.StoreInt32(&e.referenced, 0)
+		e.referenced.Store(false)
 		e.setValue(value)
 		e.ptype = etHot
 		if c.metaAdd(k, e) {
@@ -233,37 +231,75 @@ func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) {
 		return
 	}
 
-	c.mu.Lock()
+	var deletedValue *Value
-	defer c.mu.Unlock()
+	func() {
+		c.mu.Lock()
+		defer c.mu.Unlock()
 
-	e := c.blocks.Get(k)
+		e := c.blocks.Get(k)
-	if e == nil {
+		if e == nil {
-		return
+			return
-	}
+		}
-	c.metaEvict(e)
+		deletedValue = c.metaEvict(e)
-
+		c.checkConsistency()
-	c.checkConsistency()
+	}()
+	// Now that the mutex has been dropped, release the reference which will
+	// potentially free the memory associated with the previous cached value.
+	deletedValue.release()
 }
 
 // EvictFile evicts all of the cache values for the specified file.
 func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
 	fkey := key{fileKey{id, fileNum}, 0}
+	for c.evictFileRun(fkey) {
+		// Sched switch to give another goroutine an opportunity to acquire the
+		// shard mutex.
+		runtime.Gosched()
+	}
+}
+
+func (c *shard) evictFileRun(fkey key) (moreRemaining bool) {
+	// If most of the file's blocks are held in the block cache, evicting all
+	// the blocks may take a while. We don't want to block the entire cache
+	// shard, forcing concurrent readers to wait until we're finished. We drop
+	// the mutex every [blocksPerMutexAcquisition] blocks to give other
+	// goroutines an opportunity to make progress.
+	const blocksPerMutexAcquisition = 5
+	c.mu.Lock()
+
+	// Releasing a value may result in free-ing it back to the memory allocator.
+	// This can have a nontrivial cost that we'd prefer to not pay while holding
+	// the shard mutex, so we collect the evicted values in a local slice and
+	// only release them in a defer after dropping the cache mutex.
+	var obsoleteValuesAlloc [blocksPerMutexAcquisition]*Value
+	obsoleteValues := obsoleteValuesAlloc[:0]
+	defer func() {
+		c.mu.Unlock()
+		for _, v := range obsoleteValues {
+			v.release()
+		}
+	}()
+
 	blocks := c.files.Get(fkey)
 	if blocks == nil {
-		return
+		// No blocks for this file.
-	}
+		return false
-	for b, n := blocks, (*entry)(nil); ; b = n {
-		n = b.fileLink.next
-		c.metaEvict(b)
-		if b == n {
-			break
-		}
 	}
 
-	c.checkConsistency()
+	// b is the current head of the doubly linked list, and n is the entry after b.
+	for b, n := blocks, (*entry)(nil); len(obsoleteValues) < cap(obsoleteValues); b = n {
+		n = b.fileLink.next
+		obsoleteValues = append(obsoleteValues, c.metaEvict(b))
+		if b == n {
+			// b == n represents the case where b was the last entry remaining
+			// in the doubly linked list, which is why it pointed at itself. So
+			// no more entries left.
+			c.checkConsistency()
+			return false
+		}
+	}
+	// Exhausted blocksPerMutexAcquisition.
+	return true
 }
 
 func (c *shard) Free() {
@@ -274,7 +310,7 @@ func (c *shard) Free() {
 	// metaCheck call when the "invariants" build tag is specified.
 	for c.handHot != nil {
 		e := c.handHot
-		c.metaDel(c.handHot)
+		c.metaDel(c.handHot).release()
 		e.free()
 	}
 
@@ -359,12 +395,14 @@ func (c *shard) metaAdd(key key, e *entry) bool {
 
 // Remove the entry from the cache. This removes the entry from the blocks map,
 // the files map, and ensures that hand{Hot,Cold,Test} are not pointing at the
-// entry.
+// entry. Returns the deleted value that must be released, if any.
-func (c *shard) metaDel(e *entry) {
+func (c *shard) metaDel(e *entry) (deletedValue *Value) {
 	if value := e.peekValue(); value != nil {
 		value.ref.trace("metaDel")
 	}
-	e.setValue(nil)
+	// Remove the pointer to the value.
+	deletedValue = e.val
+	e.val = nil
 
 	c.blocks.Delete(e.key)
 	if entriesGoAllocated {
@@ -396,6 +434,7 @@ func (c *shard) metaDel(e *entry) {
 	} else {
 		c.files.Put(fkey, next)
 	}
+	return deletedValue
 }
 
 // Check that the specified entry is not referenced by the cache.
@@ -455,7 +494,7 @@ func (c *shard) metaCheck(e *entry) {
 	}
 }
 
-func (c *shard) metaEvict(e *entry) {
+func (c *shard) metaEvict(e *entry) (evictedValue *Value) {
 	switch e.ptype {
 	case etHot:
 		c.sizeHot -= e.size
@@ -467,9 +506,10 @@ func (c *shard) metaEvict(e *entry) {
 		c.sizeTest -= e.size
 		c.countTest--
 	}
-	c.metaDel(e)
+	evictedValue = c.metaDel(e)
 	c.metaCheck(e)
 	e.free()
+	return evictedValue
 }
 
 func (c *shard) evict() {
@@ -491,8 +531,8 @@ func (c *shard) runHandCold(countColdDebug, sizeColdDebug int64) {
 
 	e := c.handCold
 	if e.ptype == etCold {
-		if atomic.LoadInt32(&e.referenced) == 1 {
+		if e.referenced.Load() {
-			atomic.StoreInt32(&e.referenced, 0)
+			e.referenced.Store(false)
 			e.ptype = etHot
 			c.sizeCold -= e.size
 			c.countCold--
@@ -528,8 +568,8 @@ func (c *shard) runHandHot() {
 
 	e := c.handHot
 	if e.ptype == etHot {
-		if atomic.LoadInt32(&e.referenced) == 1 {
+		if e.referenced.Load() {
-			atomic.StoreInt32(&e.referenced, 0)
+			e.referenced.Store(false)
 		} else {
 			e.ptype = etCold
 			c.sizeHot -= e.size
@@ -564,7 +604,7 @@ func (c *shard) runHandTest() {
 		if c.coldTarget < 0 {
 			c.coldTarget = 0
 		}
-		c.metaDel(e)
+		c.metaDel(e).release()
 		c.metaCheck(e)
 		e.free()
 	}
@@ -587,7 +627,7 @@ type Metrics struct {
 // Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is
 // used for page replacement
 // (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In
-// order to provide better concurrency, 2 x NumCPUs shards are created, with
+// order to provide better concurrency, 4 x NumCPUs shards are created, with
 // each shard being given 1/n of the target cache size. The Clock-PRO algorithm
 // is run independently on each shard.
 //
@@ -621,9 +661,9 @@ type Metrics struct {
 // used in combination by specifying `-tags invariants,tracing`. Note that
 // "tracing" produces a significant slowdown, while "invariants" does not.
 type Cache struct {
-	refs    int64
+	refs    atomic.Int64
 	maxSize int64
-	idAlloc uint64
+	idAlloc atomic.Uint64
 	shards  []shard
 
 	// Traces recorded by Cache.trace. Used for debugging.
@@ -643,17 +683,44 @@ type Cache struct {
 //	defer c.Unref()
 //	d, err := pebble.Open(pebble.Options{Cache: c})
 func New(size int64) *Cache {
-	return newShards(size, 2*runtime.GOMAXPROCS(0))
+	// How many cache shards should we create?
+	//
+	// Note that the probability two processors will try to access the same
+	// shard at the same time increases superlinearly with the number of
+	// processors (Eg, consider the brithday problem where each CPU is a person,
+	// and each shard is a possible birthday).
+	//
+	// We could consider growing the number of shards superlinearly, but
+	// increasing the shard count may reduce the effectiveness of the caching
+	// algorithm if frequently-accessed blocks are insufficiently distributed
+	// across shards. If a shard's size is smaller than a single frequently
+	// scanned sstable, then the shard will be unable to hold the entire
+	// frequently-scanned table in memory despite other shards still holding
+	// infrequently accessed blocks.
+	//
+	// Experimentally, we've observed contention contributing to tail latencies
+	// at 2 shards per processor. For now we use 4 shards per processor,
+	// recognizing this may not be final word.
+	m := 4 * runtime.GOMAXPROCS(0)
+
+	// In tests we can use large CPU machines with small cache sizes and have
+	// many caches in existence at a time. If sharding into m shards would
+	// produce too small shards, constrain the number of shards to 4.
+	const minimumShardSize = 4 << 20 // 4 MiB
+	if m > 4 && int(size)/m < minimumShardSize {
+		m = 4
+	}
+	return newShards(size, m)
 }
 
 func newShards(size int64, shards int) *Cache {
 	c := &Cache{
-		refs:    1,
 		maxSize: size,
-		idAlloc: 1,
 		shards:  make([]shard, shards),
 	}
-	c.trace("alloc", c.refs)
+	c.refs.Store(1)
+	c.idAlloc.Store(1)
+	c.trace("alloc", c.refs.Load())
 	for i := range c.shards {
 		c.shards[i] = shard{
 			maxSize:    size / int64(len(c.shards)),
@@ -669,7 +736,7 @@ func newShards(size int64, shards int) *Cache {
 	// Note: this is a no-op if invariants are disabled or race is enabled.
 	invariants.SetFinalizer(c, func(obj interface{}) {
 		c := obj.(*Cache)
-		if v := atomic.LoadInt64(&c.refs); v != 0 {
+		if v := c.refs.Load(); v != 0 {
 			c.tr.Lock()
 			fmt.Fprintf(os.Stderr,
 				"pebble: cache (%p) has non-zero reference count: %d\n", c, v)
@@ -716,7 +783,7 @@ func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *sh
 // Ref adds a reference to the cache. The cache only remains valid as long a
 // reference is maintained to it.
 func (c *Cache) Ref() {
-	v := atomic.AddInt64(&c.refs, 1)
+	v := c.refs.Add(1)
 	if v <= 1 {
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
 	}
@@ -725,7 +792,7 @@ func (c *Cache) Ref() {
 
 // Unref releases a reference on the cache.
 func (c *Cache) Unref() {
-	v := atomic.AddInt64(&c.refs, -1)
+	v := c.refs.Add(-1)
 	c.trace("unref", v)
 	switch {
 	case v < 0:
@@ -785,14 +852,14 @@ func (c *Cache) Size() int64 {
 // manually managed. The caller MUST either add the value to the cache (via
 // Cache.Set), or release the value (via Cache.Free). Failure to do so will
 // result in a memory leak.
-func (c *Cache) Alloc(n int) *Value {
+func Alloc(n int) *Value {
 	return newValue(n)
 }
 
 // Free frees the specified value. The buffer associated with the value will
 // possibly be reused, making it invalid to use the buffer after calling
 // Free. Do not call Free on a value that has been added to the cache.
-func (c *Cache) Free(v *Value) {
+func Free(v *Value) {
 	if n := v.refs(); n > 1 {
 		panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n))
 	}
@@ -829,8 +896,8 @@ func (c *Cache) Metrics() Metrics {
 		m.Count += int64(s.blocks.Count())
 		m.Size += s.sizeHot + s.sizeCold
 		s.mu.RUnlock()
-		m.Hits += atomic.LoadInt64(&s.hits)
+		m.Hits += s.hits.Load()
-		m.Misses += atomic.LoadInt64(&s.misses)
+		m.Misses += s.misses.Load()
 	}
 	return m
 }
@@ -838,5 +905,5 @@ func (c *Cache) Metrics() Metrics {
 // NewID returns a new ID to be used as a namespace for cached file
 // blocks.
 func (c *Cache) NewID() uint64 {
-	return atomic.AddUint64(&c.idAlloc, 1)
+	return c.idAlloc.Add(1)
 }

vendor/github.com/cockroachdb/pebble/internal/cache/entry.go

@@ -4,6 +4,8 @@
 
 package cache
 
+import "sync/atomic"
+
 type entryType int8
 
 const (
@@ -56,7 +58,7 @@ type entry struct {
 	ptype entryType
 	// referenced is atomically set to indicate that this entry has been accessed
 	// since the last time one of the clock hands swept it.
-	referenced int32
+	referenced atomic.Bool
 	shard      *shard
 	// Reference count for the entry. The entry is freed when the reference count
 	// drops to zero.
@@ -137,9 +139,7 @@ func (e *entry) setValue(v *Value) {
 	}
 	old := e.val
 	e.val = v
-	if old != nil {
+	old.release()
-		old.release()
-	}
 }
 
 func (e *entry) peekValue() *Value {

vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_normal.go

@@ -10,33 +10,37 @@ package cache
 import (
 	"fmt"
 	"sync/atomic"
+
+	"github.com/cockroachdb/redact"
 )
 
 // refcnt provides an atomic reference count. This version is used when the
 // "tracing" build tag is not enabled. See refcnt_tracing.go for the "tracing"
 // enabled version.
-type refcnt int32
+type refcnt struct {
+	val atomic.Int32
+}
 
 // initialize the reference count to the specified value.
 func (v *refcnt) init(val int32) {
-	*v = refcnt(val)
+	v.val.Store(val)
 }
 
 func (v *refcnt) refs() int32 {
-	return atomic.LoadInt32((*int32)(v))
+	return v.val.Load()
 }
 
 func (v *refcnt) acquire() {
-	switch v := atomic.AddInt32((*int32)(v), 1); {
+	switch v := v.val.Add(1); {
 	case v <= 1:
-		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
+		panic(redact.Safe(fmt.Sprintf("pebble: inconsistent reference count: %d", v)))
 	}
 }
 
 func (v *refcnt) release() bool {
-	switch v := atomic.AddInt32((*int32)(v), -1); {
+	switch v := v.val.Add(-1); {
 	case v < 0:
-		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
+		panic(redact.Safe(fmt.Sprintf("pebble: inconsistent reference count: %d", v)))
 	case v == 0:
 		return true
 	default:

vendor/github.com/cockroachdb/pebble/internal/cache/refcnt_tracing.go

@@ -19,22 +19,22 @@ import (
 // debugging logic errors in manipulating the reference count. This version is
 // used when the "tracing" build tag is enabled.
 type refcnt struct {
-	val int32
+	val atomic.Int32
 	sync.Mutex
 	msgs []string
 }
 
 func (v *refcnt) init(val int32) {
-	v.val = val
+	v.val.Store(val)
 	v.trace("init")
 }
 
 func (v *refcnt) refs() int32 {
-	return atomic.LoadInt32(&v.val)
+	return v.val.Load()
 }
 
 func (v *refcnt) acquire() {
-	switch n := atomic.AddInt32(&v.val, 1); {
+	switch n := v.val.Add(1); {
 	case n <= 1:
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))
 	}
@@ -42,7 +42,7 @@ func (v *refcnt) acquire() {
 }
 
 func (v *refcnt) release() bool {
-	n := atomic.AddInt32(&v.val, -1)
+	n := v.val.Add(-1)
 	switch {
 	case n < 0:
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", n))

vendor/github.com/cockroachdb/pebble/internal/cache/value.go

@@ -40,7 +40,7 @@ func (v *Value) acquire() {
 }
 
 func (v *Value) release() {
-	if v.ref.release() {
+	if v != nil && v.ref.release() {
 		v.free()
 	}
 }

vendor/github.com/cockroachdb/pebble/internal/constants/constants.go

@@ -0,0 +1,17 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package constants
+
+const (
+	// oneIf64Bit is 1 on 64-bit platforms and 0 on 32-bit platforms.
+	oneIf64Bit = ^uint(0) >> 63
+
+	// MaxUint32OrInt returns min(MaxUint32, MaxInt), i.e
+	// - MaxUint32 on 64-bit platforms;
+	// - MaxInt on 32-bit platforms.
+	// It is used when slices are limited to Uint32 on 64-bit platforms (the
+	// length limit for slices is naturally MaxInt on 32-bit platforms).
+	MaxUint32OrInt = (1<<31)<<oneIf64Bit - 1
+)

vendor/github.com/cockroachdb/pebble/internal/humanize/humanize.go

@@ -35,11 +35,11 @@ type config struct {
 	suffix []string
 }
 
-// IEC produces human readable representations of integer values in IEC units.
+// Bytes produces human readable representations of byte values in IEC units.
-var IEC = config{1024, []string{" B", " K", " M", " G", " T", " P", " E"}}
+var Bytes = config{1024, []string{"B", "KB", "MB", "GB", "TB", "PB", "EB"}}
 
-// SI produces human readable representations of integer values in SI units.
+// Count produces human readable representations of unitless values in SI units.
-var SI = config{1000, []string{"", " K", " M", " G", " T", " P", " E"}}
+var Count = config{1000, []string{"", "K", "M", "G", "T", "P", "E"}}
 
 // Int64 produces a human readable representation of the value.
 func (c *config) Int64(s int64) FormattedString {
@@ -54,18 +54,6 @@ func (c *config) Uint64(s uint64) FormattedString {
 	return FormattedString(humanate(s, c.base, c.suffix))
 }
 
-// Int64 produces a human readable representation of the value in IEC units
-// (base 1024).
-func Int64(s int64) FormattedString {
-	return IEC.Int64(s)
-}
-
-// Uint64 produces a human readable representation of the value in IEC units
-// (base 1024).
-func Uint64(s uint64) FormattedString {
-	return IEC.Uint64(s)
-}
-
 // FormattedString represents a human readable representation of a value. It
 // implements the redact.SafeValue interface to signal that it represents a
 // a string that does not need to be redacted.

vendor/github.com/cockroachdb/pebble/internal/invalidating/iter.go

@@ -0,0 +1,162 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package invalidating
+
+import (
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/fastrand"
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// MaybeWrapIfInvariants wraps some iterators with an invalidating iterator.
+// MaybeWrapIfInvariants does nothing in non-invariant builds.
+func MaybeWrapIfInvariants(iter base.InternalIterator) base.InternalIterator {
+	if invariants.Enabled {
+		if fastrand.Uint32n(10) == 1 {
+			return NewIter(iter)
+		}
+	}
+	return iter
+}
+
+// iter tests unsafe key/value slice reuse by modifying the last
+// returned key/value to all 1s.
+type iter struct {
+	iter        base.InternalIterator
+	lastKey     *base.InternalKey
+	lastValue   base.LazyValue
+	ignoreKinds [base.InternalKeyKindMax + 1]bool
+	err         error
+}
+
+// Option configures the behavior of an invalidating iterator.
+type Option interface {
+	apply(*iter)
+}
+
+type funcOpt func(*iter)
+
+func (f funcOpt) apply(i *iter) { f(i) }
+
+// IgnoreKinds constructs an Option that configures an invalidating iterator to
+// skip trashing k/v pairs with the provided key kinds. Some iterators provided
+// key stability guarantees for specific key kinds.
+func IgnoreKinds(kinds ...base.InternalKeyKind) Option {
+	return funcOpt(func(i *iter) {
+		for _, kind := range kinds {
+			i.ignoreKinds[kind] = true
+		}
+	})
+}
+
+// NewIter constructs a new invalidating iterator that wraps the provided
+// iterator, trashing buffers for previously returned keys.
+func NewIter(originalIterator base.InternalIterator, opts ...Option) base.InternalIterator {
+	i := &iter{iter: originalIterator}
+	for _, opt := range opts {
+		opt.apply(i)
+	}
+	return i
+}
+
+func (i *iter) update(
+	key *base.InternalKey, value base.LazyValue,
+) (*base.InternalKey, base.LazyValue) {
+	i.trashLastKV()
+	if key == nil {
+		i.lastKey = nil
+		i.lastValue = base.LazyValue{}
+		return nil, base.LazyValue{}
+	}
+
+	i.lastKey = &base.InternalKey{}
+	*i.lastKey = key.Clone()
+	i.lastValue = base.LazyValue{
+		ValueOrHandle: append(make([]byte, 0, len(value.ValueOrHandle)), value.ValueOrHandle...),
+	}
+	if value.Fetcher != nil {
+		fetcher := new(base.LazyFetcher)
+		*fetcher = *value.Fetcher
+		i.lastValue.Fetcher = fetcher
+	}
+	return i.lastKey, i.lastValue
+}
+
+func (i *iter) trashLastKV() {
+	if i.lastKey == nil {
+		return
+	}
+	if i.ignoreKinds[i.lastKey.Kind()] {
+		return
+	}
+
+	if i.lastKey != nil {
+		for j := range i.lastKey.UserKey {
+			i.lastKey.UserKey[j] = 0xff
+		}
+		i.lastKey.Trailer = 0xffffffffffffffff
+	}
+	for j := range i.lastValue.ValueOrHandle {
+		i.lastValue.ValueOrHandle[j] = 0xff
+	}
+	if i.lastValue.Fetcher != nil {
+		// Not all the LazyFetcher fields are visible, so we zero out the last
+		// value's Fetcher struct entirely.
+		*i.lastValue.Fetcher = base.LazyFetcher{}
+	}
+}
+
+func (i *iter) SeekGE(key []byte, flags base.SeekGEFlags) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekGE(key, flags))
+}
+
+func (i *iter) SeekPrefixGE(
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekPrefixGE(prefix, key, flags))
+}
+
+func (i *iter) SeekLT(key []byte, flags base.SeekLTFlags) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekLT(key, flags))
+}
+
+func (i *iter) First() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.First())
+}
+
+func (i *iter) Last() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Last())
+}
+
+func (i *iter) Next() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Next())
+}
+
+func (i *iter) Prev() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Prev())
+}
+
+func (i *iter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.NextPrefix(succKey))
+}
+
+func (i *iter) Error() error {
+	if err := i.iter.Error(); err != nil {
+		return err
+	}
+	return i.err
+}
+
+func (i *iter) Close() error {
+	return i.iter.Close()
+}
+
+func (i *iter) SetBounds(lower, upper []byte) {
+	i.iter.SetBounds(lower, upper)
+}
+
+func (i *iter) String() string {
+	return i.iter.String()
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go

@@ -228,6 +228,14 @@ func (i *DefragmentingIter) SeekGE(key []byte) *Span {
 	// fragments before this span. Defragment backward to find the start of the
 	// defragmented span.
 	i.defragmentBackward()
+
+	// Defragmenting backward may have stopped because it encountered an error.
+	// If so, we must not continue so that i.iter.Error() (and thus i.Error())
+	// yields the error.
+	if i.iterSpan == nil && i.iter.Error() != nil {
+		return nil
+	}
+
 	if i.iterPos == iterPosPrev {
 		// Next once back onto the span.
 		i.iterSpan = i.iter.Next()
@@ -259,6 +267,14 @@ func (i *DefragmentingIter) SeekLT(key []byte) *Span {
 	// fragments after this span. Defragment forward to find the end of the
 	// defragmented span.
 	i.defragmentForward()
+
+	// Defragmenting forward may have stopped because it encountered an error.
+	// If so, we must not continue so that i.iter.Error() (and thus i.Error())
+	// yields the error.
+	if i.iterSpan == nil && i.iter.Error() != nil {
+		return nil
+	}
+
 	if i.iterPos == iterPosNext {
 		// Prev once back onto the span.
 		i.iterSpan = i.iter.Prev()
@@ -304,7 +320,7 @@ func (i *DefragmentingIter) Next() *Span {
 		// Next once to move onto y, defragment forward to land on the first z
 		// position.
 		i.iterSpan = i.iter.Next()
-		if invariants.Enabled && i.iterSpan == nil {
+		if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
 			panic("pebble: invariant violation: no next span while switching directions")
 		}
 		// We're now positioned on the first span that was defragmented into the
@@ -380,7 +396,7 @@ func (i *DefragmentingIter) Prev() *Span {
 		// Prev once to move onto y, defragment backward to land on the last x
 		// position.
 		i.iterSpan = i.iter.Prev()
-		if invariants.Enabled && i.iterSpan == nil {
+		if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
 			panic("pebble: invariant violation: no previous span while switching directions")
 		}
 		// We're now positioned on the last span that was defragmented into the
@@ -437,6 +453,15 @@ func (i *DefragmentingIter) defragmentForward() *Span {
 		i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
 		i.iterSpan = i.iter.Next()
 	}
+	// i.iterSpan == nil
+	//
+	// The inner iterator may return nil when it encounters an error. If there
+	// was an error, we don't know whether there is another span we should
+	// defragment or not. Return nil so that the caller knows they should check
+	// Error().
+	if i.iter.Error() != nil {
+		return nil
+	}
 	i.curr.Keys = i.keysBuf
 	return &i.curr
 }
@@ -470,6 +495,15 @@ func (i *DefragmentingIter) defragmentBackward() *Span {
 		i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
 		i.iterSpan = i.iter.Prev()
 	}
+	// i.iterSpan == nil
+	//
+	// The inner iterator may return nil when it encounters an error. If there
+	// was an error, we don't know whether there is another span we should
+	// defragment or not. Return nil so that the caller knows they should check
+	// Error().
+	if i.iter.Error() != nil {
+		return nil
+	}
 	i.curr.Keys = i.keysBuf
 	return &i.curr
 }

vendor/github.com/cockroachdb/pebble/internal/keyspan/filter.go

@@ -4,6 +4,8 @@
 
 package keyspan
 
+import "github.com/cockroachdb/pebble/internal/base"
+
 // FilterFunc defines a transform from the input Span into the output Span. The
 // function returns true if the Span should be returned by the iterator, and
 // false if the Span should be skipped. The FilterFunc is permitted to mutate
@@ -21,6 +23,7 @@ type FilterFunc func(in *Span, out *Span) (keep bool)
 type filteringIter struct {
 	iter     FragmentIterator
 	filterFn FilterFunc
+	cmp      base.Compare
 
 	// span is a mutable Span passed to the filterFn. The filterFn is free to
 	// mutate this Span. The slice of Keys in the Span is reused with every call
@@ -32,18 +35,32 @@ var _ FragmentIterator = (*filteringIter)(nil)
 
 // Filter returns a new filteringIter that will filter the Spans from the
 // provided child iterator using the provided FilterFunc.
-func Filter(iter FragmentIterator, filter FilterFunc) FragmentIterator {
+func Filter(iter FragmentIterator, filter FilterFunc, cmp base.Compare) FragmentIterator {
-	return &filteringIter{iter: iter, filterFn: filter}
+	return &filteringIter{iter: iter, filterFn: filter, cmp: cmp}
 }
 
 // SeekGE implements FragmentIterator.
 func (i *filteringIter) SeekGE(key []byte) *Span {
-	return i.filter(i.iter.SeekGE(key), +1)
+	span := i.filter(i.iter.SeekGE(key), +1)
+	// i.filter could return a span that's less than key, _if_ the filterFunc
+	// (which has no knowledge of the seek key) mutated the span to end at a key
+	// less than or equal to `key`. Detect this case and next/invalidate the iter.
+	if span != nil && i.cmp(span.End, key) <= 0 {
+		return i.Next()
+	}
+	return span
 }
 
 // SeekLT implements FragmentIterator.
 func (i *filteringIter) SeekLT(key []byte) *Span {
-	return i.filter(i.iter.SeekLT(key), -1)
+	span := i.filter(i.iter.SeekLT(key), -1)
+	// i.filter could return a span that's >= key, _if_ the filterFunc (which has
+	// no knowledge of the seek key) mutated the span to start at a key greater
+	// than or equal to `key`. Detect this case and prev/invalidate the iter.
+	if span != nil && i.cmp(span.Start, key) >= 0 {
+		return i.Prev()
+	}
+	return span
 }
 
 // First implements FragmentIterator.

vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go

@@ -51,6 +51,8 @@ type FragmentIterator interface {
 	Prev() *Span
 
 	// Error returns any accumulated error.
+	//
+	// TODO(jackson): Lift errors into return values on the positioning methods.
 	Error() error
 
 	// Close closes the iterator and returns any accumulated error. Exhausting
@@ -70,9 +72,6 @@ type SpanIterOptions struct {
 	// RangeKeyFilters can be used to avoid scanning tables and blocks in tables
 	// when iterating over range keys.
 	RangeKeyFilters []base.BlockPropertyFilter
-	// Level specifies the level where this sstable is being read. Must be
-	// specified for foreign (i.e. shared not-created-by-this-instance) sstables.
-	Level manifest.Level
 }
 
 // Iter is an iterator over a set of fragmented spans.

vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go

@@ -48,102 +48,3 @@ func VisibleTransform(snapshot uint64) Transformer {
 		return nil
 	})
 }
-
-// TransformerIter is a FragmentIterator that applies a Transformer on all
-// returned keys. Used for when a caller needs to apply a transformer on an
-// iterator but does not otherwise need the mergingiter's merging ability.
-type TransformerIter struct {
-	FragmentIterator
-
-	// Transformer is applied on every Span returned by this iterator.
-	Transformer Transformer
-	// Comparer in use for this keyspace.
-	Compare base.Compare
-
-	span Span
-	err  error
-}
-
-func (t *TransformerIter) applyTransform(span *Span) *Span {
-	t.span = Span{
-		Start: t.span.Start[:0],
-		End:   t.span.End[:0],
-		Keys:  t.span.Keys[:0],
-	}
-	if err := t.Transformer.Transform(t.Compare, *span, &t.span); err != nil {
-		t.err = err
-		return nil
-	}
-	return &t.span
-}
-
-// SeekGE implements the FragmentIterator interface.
-func (t *TransformerIter) SeekGE(key []byte) *Span {
-	span := t.FragmentIterator.SeekGE(key)
-	if span == nil {
-		return nil
-	}
-	return t.applyTransform(span)
-}
-
-// SeekLT implements the FragmentIterator interface.
-func (t *TransformerIter) SeekLT(key []byte) *Span {
-	span := t.FragmentIterator.SeekLT(key)
-	if span == nil {
-		return nil
-	}
-	return t.applyTransform(span)
-}
-
-// First implements the FragmentIterator interface.
-func (t *TransformerIter) First() *Span {
-	span := t.FragmentIterator.First()
-	if span == nil {
-		return nil
-	}
-	return t.applyTransform(span)
-}
-
-// Last implements the FragmentIterator interface.
-func (t *TransformerIter) Last() *Span {
-	span := t.FragmentIterator.Last()
-	if span == nil {
-		return nil
-	}
-	return t.applyTransform(span)
-}
-
-// Next implements the FragmentIterator interface.
-func (t *TransformerIter) Next() *Span {
-	span := t.FragmentIterator.Next()
-	if span == nil {
-		return nil
-	}
-	return t.applyTransform(span)
-}
-
-// Prev implements the FragmentIterator interface.
-func (t *TransformerIter) Prev() *Span {
-	span := t.FragmentIterator.Prev()
-	if span == nil {
-		return nil
-	}
-	return t.applyTransform(span)
-}
-
-// Error implements the FragmentIterator interface.
-func (t *TransformerIter) Error() error {
-	if t.err != nil {
-		return t.err
-	}
-	return t.FragmentIterator.Error()
-}
-
-// Close implements the FragmentIterator interface.
-func (t *TransformerIter) Close() error {
-	err := t.FragmentIterator.Close()
-	if err != nil {
-		return err
-	}
-	return t.err
-}

vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go

@@ -14,7 +14,7 @@ func Truncate(
 	iter FragmentIterator,
 	lower, upper []byte,
 	start, end *base.InternalKey,
-	panicOnPartialOverlap bool,
+	panicOnUpperTruncate bool,
 ) FragmentIterator {
 	return Filter(iter, func(in *Span, out *Span) (keep bool) {
 		out.Start, out.End = in.Start, in.End
@@ -57,7 +57,6 @@ func Truncate(
 		var truncated bool
 		// Truncate the bounds to lower and upper.
 		if cmp(in.Start, lower) < 0 {
-			truncated = true
 			out.Start = lower
 		}
 		if cmp(in.End, upper) > 0 {
@@ -65,10 +64,10 @@ func Truncate(
 			out.End = upper
 		}
 
-		if panicOnPartialOverlap && truncated {
+		if panicOnUpperTruncate && truncated {
-			panic("pebble: bounds should not be truncated")
+			panic("pebble: upper bound should not be truncated")
 		}
 
 		return !out.Empty() && cmp(out.Start, out.End) < 0
-	})
+	}, cmp)
 }

vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go

@@ -108,7 +108,7 @@ type annotation struct {
 }
 
 type leafNode struct {
-	ref   int32
+	ref   atomic.Int32
 	count int16
 	leaf  bool
 	// subtreeCount holds the count of files in the entire subtree formed by
@@ -139,13 +139,13 @@ func leafToNode(ln *leafNode) *node {
 func newLeafNode() *node {
 	n := leafToNode(new(leafNode))
 	n.leaf = true
-	n.ref = 1
+	n.ref.Store(1)
 	return n
 }
 
 func newNode() *node {
 	n := new(node)
-	n.ref = 1
+	n.ref.Store(1)
 	return n
 }
 
@@ -159,7 +159,7 @@ func newNode() *node {
 // When a node is cloned, the provided pointer will be redirected to the new
 // mutable node.
 func mut(n **node) *node {
-	if atomic.LoadInt32(&(*n).ref) == 1 {
+	if (*n).ref.Load() == 1 {
 		// Exclusive ownership. Can mutate in place.
 
 		// Whenever a node will be mutated, reset its annotations to be marked
@@ -186,7 +186,7 @@ func mut(n **node) *node {
 
 // incRef acquires a reference to the node.
 func (n *node) incRef() {
-	atomic.AddInt32(&n.ref, 1)
+	n.ref.Add(1)
 }
 
 // decRef releases a reference to the node. If requested, the method will unref
@@ -195,8 +195,8 @@ func (n *node) incRef() {
 // new nodes pass contentsToo=false to preserve existing reference counts during
 // operations that should yield a net-zero change to descendant refcounts.
 // When a node is released, its contained files are dereferenced.
-func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
+func (n *node) decRef(contentsToo bool, obsolete *[]*FileBacking) {
-	if atomic.AddInt32(&n.ref, -1) > 0 {
+	if n.ref.Add(-1) > 0 {
 		// Other references remain. Can't free.
 		return
 	}
@@ -215,7 +215,13 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
 				if obsolete == nil {
 					panic(fmt.Sprintf("file metadata %s dereferenced to zero during tree mutation", f.FileNum))
 				}
-				*obsolete = append(*obsolete, f)
+				// Reference counting is performed on the FileBacking. In the case
+				// of a virtual sstable, this reference counting is performed on
+				// a FileBacking which is shared by every single virtual sstable
+				// with the same backing sstable. If the reference count hits 0,
+				// then we know that the FileBacking won't be required by any
+				// sstable in Pebble, and that the backing sstable can be deleted.
+				*obsolete = append(*obsolete, f.FileBacking)
 			}
 		}
 		if !n.leaf {
@@ -761,8 +767,8 @@ type btree struct {
 
 // Release dereferences and clears the root node of the btree, removing all
 // items from the btree. In doing so, it decrements contained file counts.
-// It returns a slice of newly obsolete files, if any.
+// It returns a slice of newly obsolete backing files, if any.
-func (t *btree) Release() (obsolete []*FileMetadata) {
+func (t *btree) Release() (obsolete []*FileBacking) {
 	if t.root != nil {
 		t.root.decRef(true /* contentsToo */, &obsolete)
 		t.root = nil

vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go

@@ -712,7 +712,7 @@ func (s *L0Sublevels) calculateFlushSplitKeys(flushSplitMaxBytes int64) {
 // InitCompactingFileInfo initializes internal flags relating to compacting
 // files. Must be called after sublevel initialization.
 //
-// Requires DB.mu to be held.
+// Requires DB.mu *and* the manifest lock to be held.
 func (s *L0Sublevels) InitCompactingFileInfo(inProgress []L0Compaction) {
 	for i := range s.orderedIntervals {
 		s.orderedIntervals[i].compactingFileCount = 0
@@ -735,6 +735,13 @@ func (s *L0Sublevels) InitCompactingFileInfo(inProgress []L0Compaction) {
 		if !f.IsCompacting() {
 			continue
 		}
+		if invariants.Enabled {
+			if s.cmp(s.orderedIntervals[f.minIntervalIndex].startKey.key, f.Smallest.UserKey) != 0 || s.cmp(s.orderedIntervals[f.maxIntervalIndex+1].startKey.key, f.Largest.UserKey) != 0 {
+				panic(fmt.Sprintf("file %s has inconsistent L0 Sublevel interval bounds: %s-%s, %s-%s", f.FileNum,
+					s.orderedIntervals[f.minIntervalIndex].startKey.key, s.orderedIntervals[f.maxIntervalIndex+1].startKey.key,
+					f.Smallest.UserKey, f.Largest.UserKey))
+			}
+		}
 		for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ {
 			interval := &s.orderedIntervals[i]
 			interval.compactingFileCount++

vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go

@@ -15,20 +15,28 @@ import (
 // LevelMetadata contains metadata for all of the files within
 // a level of the LSM.
 type LevelMetadata struct {
-	level int
+	level     int
-	tree  btree
+	totalSize uint64
+	// NumVirtual is the number of virtual sstables in the level.
+	NumVirtual uint64
+	// VirtualSize is the size of the virtual sstables in the level.
+	VirtualSize uint64
+	tree        btree
 }
 
 // clone makes a copy of the level metadata, implicitly increasing the ref
 // count of every file contained within lm.
 func (lm *LevelMetadata) clone() LevelMetadata {
 	return LevelMetadata{
-		level: lm.level,
+		level:       lm.level,
-		tree:  lm.tree.Clone(),
+		totalSize:   lm.totalSize,
+		NumVirtual:  lm.NumVirtual,
+		VirtualSize: lm.VirtualSize,
+		tree:        lm.tree.Clone(),
 	}
 }
 
-func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
+func (lm *LevelMetadata) release() (obsolete []*FileBacking) {
 	return lm.tree.Release()
 }
 
@@ -40,6 +48,13 @@ func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetad
 	var lm LevelMetadata
 	lm.level = level
 	lm.tree, _ = makeBTree(bcmp, files)
+	for _, f := range files {
+		lm.totalSize += f.Size
+		if f.Virtual {
+			lm.NumVirtual++
+			lm.VirtualSize += f.Size
+		}
+	}
 	return lm
 }
 
@@ -52,6 +67,27 @@ func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) {
 	return t, newLevelSlice(t.Iter())
 }
 
+func (lm *LevelMetadata) insert(f *FileMetadata) error {
+	if err := lm.tree.Insert(f); err != nil {
+		return err
+	}
+	lm.totalSize += f.Size
+	if f.Virtual {
+		lm.NumVirtual++
+		lm.VirtualSize += f.Size
+	}
+	return nil
+}
+
+func (lm *LevelMetadata) remove(f *FileMetadata) bool {
+	lm.totalSize -= f.Size
+	if f.Virtual {
+		lm.NumVirtual--
+		lm.VirtualSize -= f.Size
+	}
+	return lm.tree.Delete(f)
+}
+
 // Empty indicates whether there are any files in the level.
 func (lm *LevelMetadata) Empty() bool {
 	return lm.tree.Count() == 0
@@ -62,6 +98,11 @@ func (lm *LevelMetadata) Len() int {
 	return lm.tree.Count()
 }
 
+// Size returns the cumulative size of all the files within the level.
+func (lm *LevelMetadata) Size() uint64 {
+	return lm.totalSize
+}
+
 // Iter constructs a LevelIterator over the entire level.
 func (lm *LevelMetadata) Iter() LevelIterator {
 	return LevelIterator{iter: lm.tree.Iter()}
@@ -280,6 +321,32 @@ func (ls *LevelSlice) SizeSum() uint64 {
 	return sum
 }
 
+// NumVirtual returns the number of virtual sstables in the level. Its runtime is
+// linear in the length of the slice.
+func (ls *LevelSlice) NumVirtual() uint64 {
+	var n uint64
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		if f.Virtual {
+			n++
+		}
+	}
+	return n
+}
+
+// VirtualSizeSum returns the sum of the sizes of the virtual sstables in the
+// level.
+func (ls *LevelSlice) VirtualSizeSum() uint64 {
+	var sum uint64
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		if f.Virtual {
+			sum += f.Size
+		}
+	}
+	return sum
+}
+
 // Reslice constructs a new slice backed by the same underlying level, with
 // new start and end positions. Reslice invokes the provided function, passing
 // two LevelIterators: one positioned to i's inclusive start and one

vendor/github.com/cockroachdb/pebble/internal/manifest/version.go

@@ -49,6 +49,9 @@ type TableStats struct {
 	// The number of point and range deletion entries in the table.
 	NumDeletions uint64
 	// NumRangeKeySets is the total number of range key sets in the table.
+	//
+	// NB: If there's a chance that the sstable contains any range key sets,
+	// then NumRangeKeySets must be > 0.
 	NumRangeKeySets uint64
 	// Estimate of the total disk space that may be dropped by this table's
 	// point deletions by compacting them.
@@ -169,22 +172,13 @@ type FileMetadata struct {
 	// FileNum is the file number.
 	//
 	// INVARIANT: when !FileMetadata.Virtual, FileNum == FileBacking.DiskFileNum.
-	//
-	// TODO(bananabrick): Consider creating separate types for
-	// FileMetadata.FileNum and FileBacking.FileNum. FileNum is used both as
-	// an indentifier for the FileMetadata in Pebble, and also as a handle to
-	// perform reads and writes. We should ensure through types that
-	// FileMetadata.FileNum isn't used to perform reads, and that
-	// FileBacking.FileNum isn't used as an identifier for the FileMetadata.
 	FileNum base.FileNum
 	// Size is the size of the file, in bytes. Size is an approximate value for
 	// virtual sstables.
 	//
-	// INVARIANT: when !FileMetadata.Virtual, Size == FileBacking.Size.
+	// INVARIANTS:
-	//
+	// - When !FileMetadata.Virtual, Size == FileBacking.Size.
-	// TODO(bananabrick): Size is currently used in metrics, and for many key
+	// - Size should be non-zero. Size 0 virtual sstables must not be created.
-	// Pebble level heuristics. Make sure that the heuristics will still work
-	// appropriately with an approximate value of size.
 	Size uint64
 	// File creation time in seconds since the epoch (1970-01-01 00:00:00
 	// UTC). For ingested sstables, this corresponds to the time the file was
@@ -227,10 +221,6 @@ type FileMetadata struct {
 	// we'd have to write virtual sstable stats to the version edit.
 	Stats TableStats
 
-	// Deleted is set to true if a VersionEdit gets installed that has deleted
-	// this file. Protected by the manifest lock (see versionSet.logLock()).
-	Deleted bool
-
 	// For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and
 	// pick L0 compactions. Only accurate for the most recent Version.
 	SubLevel         int
@@ -292,6 +282,34 @@ type PhysicalFileMeta struct {
 // VirtualFileMeta is used by functions which want a guarantee that their input
 // belongs to a virtual sst and not a physical sst.
 //
+// A VirtualFileMeta inherits all the same fields as a FileMetadata. These
+// fields have additional invariants imposed on them, and/or slightly varying
+// meanings:
+//   - Smallest and Largest (and their counterparts
+//     {Smallest, Largest}{Point,Range}Key) remain tight bounds that represent a
+//     key at that exact bound. We make the effort to determine the next smallest
+//     or largest key in an sstable after virtualizing it, to maintain this
+//     tightness. If the largest is a sentinel key (IsExclusiveSentinel()), it
+//     could mean that a rangedel or range key ends at that user key, or has been
+//     truncated to that user key.
+//   - One invariant is that if a rangedel or range key is truncated on its
+//     upper bound, the virtual sstable *must* have a rangedel or range key
+//     sentinel key as its upper bound. This is because truncation yields
+//     an exclusive upper bound for the rangedel/rangekey, and if there are
+//     any points at that exclusive upper bound within the same virtual
+//     sstable, those could get uncovered by this truncation. We enforce this
+//     invariant in calls to keyspan.Truncate.
+//   - Size is an estimate of the size of the virtualized portion of this sstable.
+//     The underlying file's size is stored in FileBacking.Size, though it could
+//     also be estimated or could correspond to just the referenced portion of
+//     a file (eg. if the file originated on another node).
+//   - Size must be > 0.
+//   - SmallestSeqNum and LargestSeqNum are loose bounds for virtual sstables.
+//     This means that all keys in the virtual sstable must have seqnums within
+//     [SmallestSeqNum, LargestSeqNum], however there's no guarantee that there's
+//     a key with a seqnum at either of the bounds. Calculating tight seqnum
+//     bounds would be too expensive and deliver little value.
+//
 // NB: This type should only be constructed by calling FileMetadata.VirtualMeta.
 type VirtualFileMeta struct {
 	*FileMetadata
@@ -589,6 +607,13 @@ func (m *FileMetadata) Overlaps(cmp Compare, start []byte, end []byte, exclusive
 	return true
 }
 
+// ContainedWithinSpan returns true if the file key range completely overlaps with the
+// given range ("end" is assumed to exclusive).
+func (m *FileMetadata) ContainedWithinSpan(cmp Compare, start, end []byte) bool {
+	lowerCmp, upperCmp := cmp(m.Smallest.UserKey, start), cmp(m.Largest.UserKey, end)
+	return lowerCmp >= 0 && (upperCmp < 0 || (upperCmp == 0 && m.Largest.IsExclusiveSentinel()))
+}
+
 // ContainsKeyType returns whether or not the file contains keys of the provided
 // type.
 func (m *FileMetadata) ContainsKeyType(kt KeyType) bool {
@@ -686,6 +711,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
 	if !verbose {
 		return b.String()
 	}
+	fmt.Fprintf(&b, " seqnums:[%d-%d]", m.SmallestSeqNum, m.LargestSeqNum)
 	if m.HasPointKeys {
 		fmt.Fprintf(&b, " points:[%s-%s]",
 			m.SmallestPointKey.Pretty(format), m.LargestPointKey.Pretty(format))
@@ -701,7 +727,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string {
 // representation.
 func ParseFileMetadataDebug(s string) (*FileMetadata, error) {
 	// Split lines of the form:
-	//  000000:[a#0,SET-z#0,SET] points:[...] ranges:[...]
+	//  000000:[a#0,SET-z#0,SET] seqnums:[5-5] points:[...] ranges:[...]
 	fields := strings.FieldsFunc(s, func(c rune) bool {
 		switch c {
 		case ':', '[', '-', ']':
@@ -716,6 +742,19 @@ func ParseFileMetadataDebug(s string) (*FileMetadata, error) {
 	m := &FileMetadata{}
 	for len(fields) > 0 {
 		prefix := fields[0]
+		if prefix == "seqnums" {
+			smallestSeqNum, err := strconv.ParseUint(fields[1], 10, 64)
+			if err != nil {
+				return m, errors.Newf("malformed input: %s: %s", s, err)
+			}
+			largestSeqNum, err := strconv.ParseUint(fields[2], 10, 64)
+			if err != nil {
+				return m, errors.Newf("malformed input: %s: %s", s, err)
+			}
+			m.SmallestSeqNum, m.LargestSeqNum = smallestSeqNum, largestSeqNum
+			fields = fields[3:]
+			continue
+		}
 		smallest := base.ParsePrettyInternalKey(fields[1])
 		largest := base.ParsePrettyInternalKey(fields[2])
 		switch prefix {
@@ -983,6 +1022,9 @@ func NewVersion(
 		} else {
 			v.Levels[l].tree.cmp = btreeCmpSmallestKey(cmp)
 		}
+		for _, f := range files[l] {
+			v.Levels[l].totalSize += f.Size
+		}
 	}
 	if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
 		panic(err)
@@ -1015,7 +1057,7 @@ func NewVersion(
 // key in a higher level table that has both the same user key and a higher
 // sequence number.
 type Version struct {
-	refs int32
+	refs atomic.Int32
 
 	// The level 0 sstables are organized in a series of sublevels. Similar to
 	// the seqnum invariant in normal levels, there is no internal key in a
@@ -1145,12 +1187,12 @@ func ParseVersionDebug(
 
 // Refs returns the number of references to the version.
 func (v *Version) Refs() int32 {
-	return atomic.LoadInt32(&v.refs)
+	return v.refs.Load()
 }
 
 // Ref increments the version refcount.
 func (v *Version) Ref() {
-	atomic.AddInt32(&v.refs, 1)
+	v.refs.Add(1)
 }
 
 // Unref decrements the version refcount. If the last reference to the version
@@ -1158,16 +1200,11 @@ func (v *Version) Ref() {
 // Deleted callback is invoked. Requires that the VersionList mutex is NOT
 // locked.
 func (v *Version) Unref() {
-	if atomic.AddInt32(&v.refs, -1) == 0 {
+	if v.refs.Add(-1) == 0 {
 		l := v.list
 		l.mu.Lock()
 		l.Remove(v)
-		obsolete := v.unrefFiles()
+		v.Deleted(v.unrefFiles())
-		fileBacking := make([]*FileBacking, len(obsolete))
-		for i, f := range obsolete {
-			fileBacking[i] = f.FileBacking
-		}
-		v.Deleted(fileBacking)
 		l.mu.Unlock()
 	}
 }
@@ -1177,19 +1214,14 @@ func (v *Version) Unref() {
 // the Deleted callback is invoked. Requires that the VersionList mutex is
 // already locked.
 func (v *Version) UnrefLocked() {
-	if atomic.AddInt32(&v.refs, -1) == 0 {
+	if v.refs.Add(-1) == 0 {
 		v.list.Remove(v)
-		obsolete := v.unrefFiles()
+		v.Deleted(v.unrefFiles())
-		fileBacking := make([]*FileBacking, len(obsolete))
-		for i, f := range obsolete {
-			fileBacking[i] = f.FileBacking
-		}
-		v.Deleted(fileBacking)
 	}
 }
 
-func (v *Version) unrefFiles() []*FileMetadata {
+func (v *Version) unrefFiles() []*FileBacking {
-	var obsolete []*FileMetadata
+	var obsolete []*FileBacking
 	for _, lm := range v.Levels {
 		obsolete = append(obsolete, lm.release()...)
 	}
@@ -1321,16 +1353,20 @@ func (v *Version) Overlaps(
 // CheckOrdering checks that the files are consistent with respect to
 // increasing file numbers (for level 0 files) and increasing and non-
 // overlapping internal key ranges (for level non-0 files).
-func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error {
+func (v *Version) CheckOrdering(
+	cmp Compare, format base.FormatKey, order OrderingInvariants,
+) error {
 	for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
 		sublevelIter := v.L0SublevelFiles[sublevel].Iter()
-		if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter); err != nil {
+		// Sublevels have NEVER allowed split user keys, so we can pass
+		// ProhibitSplitUserKeys.
+		if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter, ProhibitSplitUserKeys); err != nil {
 			return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
 		}
 	}
 
 	for level, lm := range v.Levels {
-		if err := CheckOrdering(cmp, format, Level(level), lm.Iter()); err != nil {
+		if err := CheckOrdering(cmp, format, Level(level), lm.Iter(), order); err != nil {
 			return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
 		}
 	}
@@ -1399,10 +1435,34 @@ func (l *VersionList) Remove(v *Version) {
 	v.list = nil // avoid memory leaks
 }
 
+// OrderingInvariants dictates the file ordering invariants active.
+type OrderingInvariants int8
+
+const (
+	// ProhibitSplitUserKeys indicates that adjacent files within a level cannot
+	// contain the same user key.
+	ProhibitSplitUserKeys OrderingInvariants = iota
+	// AllowSplitUserKeys indicates that adjacent files within a level may
+	// contain the same user key. This is only allowed by historical format
+	// major versions.
+	//
+	// TODO(jackson): Remove.
+	AllowSplitUserKeys
+)
+
 // CheckOrdering checks that the files are consistent with respect to
 // seqnums (for level 0 files -- see detailed comment below) and increasing and non-
 // overlapping internal key ranges (for non-level 0 files).
-func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelIterator) error {
+//
+// The ordering field may be passed AllowSplitUserKeys to allow adjacent files that are both
+// inclusive of the same user key. Pebble no longer creates version edits
+// installing such files, and Pebble databases with sufficiently high format
+// major version should no longer have any such files within their LSM.
+// TODO(jackson): Remove AllowSplitUserKeys when we remove support for the
+// earlier format major versions.
+func CheckOrdering(
+	cmp Compare, format base.FormatKey, level Level, files LevelIterator, ordering OrderingInvariants,
+) error {
 	// The invariants to check for L0 sublevels are the same as the ones to
 	// check for all other levels. However, if L0 is not organized into
 	// sublevels, or if all L0 files are being passed in, we do the legacy L0
@@ -1480,11 +1540,29 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelI
 						prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
 						f.Smallest.Pretty(format), f.Largest.Pretty(format))
 				}
-				if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
+
-					return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
+				// What's considered "overlapping" is dependent on the format
-						errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
+				// major version. If ordering=ProhibitSplitUserKeys, then both
-						prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
+				// files cannot contain keys with the same user keys. If the
-						f.Smallest.Pretty(format), f.Largest.Pretty(format))
+				// bounds have the same user key, the previous file's boundary
+				// must have a Trailer indicating that it's exclusive.
+				switch ordering {
+				case AllowSplitUserKeys:
+					if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
+						return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
+							errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
+							prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
+							f.Smallest.Pretty(format), f.Largest.Pretty(format))
+					}
+				case ProhibitSplitUserKeys:
+					if v := cmp(prev.Largest.UserKey, f.Smallest.UserKey); v > 0 || (v == 0 && !prev.Largest.IsExclusiveSentinel()) {
+						return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
+							errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
+							prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
+							f.Smallest.Pretty(format), f.Largest.Pretty(format))
+					}
+				default:
+					panic("unreachable")
 				}
 			}
 		}

vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go

@@ -450,8 +450,7 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 	return nil
 }
 
-// String implements fmt.Stringer for a VersionEdit.
+func (v *VersionEdit) string(verbose bool, fmtKey base.FormatKey) string {
-func (v *VersionEdit) String() string {
 	var buf bytes.Buffer
 	if v.ComparerName != "" {
 		fmt.Fprintf(&buf, "  comparer:     %s", v.ComparerName)
@@ -482,7 +481,12 @@ func (v *VersionEdit) String() string {
 		fmt.Fprintf(&buf, "  deleted:       L%d %s\n", df.Level, df.FileNum)
 	}
 	for _, nf := range v.NewFiles {
-		fmt.Fprintf(&buf, "  added:         L%d %s", nf.Level, nf.Meta.String())
+		fmt.Fprintf(&buf, "  added:         L%d", nf.Level)
+		if verbose {
+			fmt.Fprintf(&buf, " %s", nf.Meta.DebugString(fmtKey, true /* verbose */))
+		} else {
+			fmt.Fprintf(&buf, " %s", nf.Meta.String())
+		}
 		if nf.Meta.CreationTime != 0 {
 			fmt.Fprintf(&buf, " (%s)",
 				time.Unix(nf.Meta.CreationTime, 0).UTC().Format(time.RFC3339))
@@ -492,6 +496,16 @@ func (v *VersionEdit) String() string {
 	return buf.String()
 }
 
+// DebugString is a more verbose version of String(). Use this in tests.
+func (v *VersionEdit) DebugString(fmtKey base.FormatKey) string {
+	return v.string(true /* verbose */, fmtKey)
+}
+
+// String implements fmt.Stringer for a VersionEdit.
+func (v *VersionEdit) String() string {
+	return v.string(false /* verbose */, base.DefaultFormatter)
+}
+
 // Encode encodes an edit to the specified writer.
 func (v *VersionEdit) Encode(w io.Writer) error {
 	e := versionEditEncoder{new(bytes.Buffer)}
@@ -769,6 +783,12 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 		b.AddedFileBacking = make(map[base.DiskFileNum]*FileBacking)
 	}
 	for _, fb := range ve.CreatedBackingTables {
+		if _, ok := b.AddedFileBacking[fb.DiskFileNum]; ok {
+			// There is already a FileBacking associated with fb.DiskFileNum.
+			// This should never happen. There must always be only one FileBacking
+			// associated with a backing sstable.
+			panic(fmt.Sprintf("pebble: duplicate file backing %s", fb.DiskFileNum.String()))
+		}
 		b.AddedFileBacking[fb.DiskFileNum] = fb
 	}
 
@@ -832,6 +852,9 @@ func AccumulateIncompleteAndApplySingleVE(
 	flushSplitBytes int64,
 	readCompactionRate int64,
 	backingStateMap map[base.DiskFileNum]*FileBacking,
+	addBackingFunc func(*FileBacking),
+	removeBackingFunc func(base.DiskFileNum),
+	orderingInvariants OrderingInvariants,
 ) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) {
 	if len(ve.RemovedBackingTables) != 0 {
 		panic("pebble: invalid incomplete version edit")
@@ -843,14 +866,14 @@ func AccumulateIncompleteAndApplySingleVE(
 	}
 	zombies = make(map[base.DiskFileNum]uint64)
 	v, err := b.Apply(
-		curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies,
+		curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies, orderingInvariants,
 	)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	for _, s := range b.AddedFileBacking {
-		backingStateMap[s.DiskFileNum] = s
+		addBackingFunc(s)
 	}
 
 	for fileNum := range zombies {
@@ -861,10 +884,9 @@ func AccumulateIncompleteAndApplySingleVE(
 			ve.RemovedBackingTables = append(
 				ve.RemovedBackingTables, fileNum,
 			)
-			delete(backingStateMap, fileNum)
+			removeBackingFunc(fileNum)
 		}
 	}
-
 	return v, zombies, nil
 }
 
@@ -885,6 +907,7 @@ func (b *BulkVersionEdit) Apply(
 	flushSplitBytes int64,
 	readCompactionRate int64,
 	zombies map[base.DiskFileNum]uint64,
+	orderingInvariants OrderingInvariants,
 ) (*Version, error) {
 	addZombie := func(state *FileBacking) {
 		if zombies != nil {
@@ -953,7 +976,7 @@ func (b *BulkVersionEdit) Apply(
 		// level.
 
 		for _, f := range deletedFilesMap {
-			if obsolete := v.Levels[level].tree.Delete(f); obsolete {
+			if obsolete := v.Levels[level].remove(f); obsolete {
 				// Deleting a file from the B-Tree may decrement its
 				// reference count. However, because we cloned the
 				// previous level's B-Tree, this should never result in a
@@ -962,7 +985,7 @@ func (b *BulkVersionEdit) Apply(
 				return nil, err
 			}
 			if f.HasRangeKeys {
-				if obsolete := v.RangeKeyLevels[level].tree.Delete(f); obsolete {
+				if obsolete := v.RangeKeyLevels[level].remove(f); obsolete {
 					// Deleting a file from the B-Tree may decrement its
 					// reference count. However, because we cloned the
 					// previous level's B-Tree, this should never result in a
@@ -1015,7 +1038,7 @@ func (b *BulkVersionEdit) Apply(
 			f.AllowedSeeks.Store(allowedSeeks)
 			f.InitAllowedSeeks = allowedSeeks
 
-			err := lm.tree.Insert(f)
+			err := lm.insert(f)
 			// We're adding this file to the new version, so increment the
 			// latest refs count.
 			f.LatestRef()
@@ -1023,7 +1046,7 @@ func (b *BulkVersionEdit) Apply(
 				return nil, errors.Wrap(err, "pebble")
 			}
 			if f.HasRangeKeys {
-				err = lmRange.tree.Insert(f)
+				err = lmRange.insert(f)
 				if err != nil {
 					return nil, errors.Wrap(err, "pebble")
 				}
@@ -1068,7 +1091,7 @@ func (b *BulkVersionEdit) Apply(
 			} else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
 				return nil, errors.Wrap(err, "pebble: internal error")
 			}
-			if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil {
+			if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter(), orderingInvariants); err != nil {
 				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 			continue
@@ -1089,7 +1112,7 @@ func (b *BulkVersionEdit) Apply(
 					end.Prev()
 				}
 			})
-			if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil {
+			if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter(), orderingInvariants); err != nil {
 				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 		}

vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go

@@ -6,7 +6,6 @@ package rangekey
 
 import (
 	"bytes"
-	"fmt"
 	"math"
 	"sort"
 
@@ -53,15 +52,15 @@ import (
 //	                        │
 //	                        ╰── <?>.Next
 type UserIteratorConfig struct {
-	snapshot   uint64
+	snapshot     uint64
-	comparer   *base.Comparer
+	comparer     *base.Comparer
-	miter      keyspan.MergingIter
+	miter        keyspan.MergingIter
-	biter      keyspan.BoundedIter
+	biter        keyspan.BoundedIter
-	diter      keyspan.DefragmentingIter
+	diter        keyspan.DefragmentingIter
-	liters     [manifest.NumLevels]keyspan.LevelIter
+	liters       [manifest.NumLevels]keyspan.LevelIter
-	litersUsed int
+	litersUsed   int
-	onlySets   bool
+	internalKeys bool
-	bufs       *Buffers
+	bufs         *Buffers
 }
 
 // Buffers holds various buffers used for range key iteration. They're exposed
@@ -80,9 +79,10 @@ func (bufs *Buffers) PrepareForReuse() {
 
 // Init initializes the range key iterator stack for user iteration. The
 // resulting fragment iterator applies range key semantics, defragments spans
-// according to their user-observable state and, if onlySets = true, removes all
+// according to their user-observable state and, if !internalKeys, removes all
 // Keys other than RangeKeySets describing the current state of range keys. The
-// resulting spans contain Keys sorted by Suffix.
+// resulting spans contain Keys sorted by suffix (unless internalKeys is true,
+// in which case they remain sorted by trailer descending).
 //
 // The snapshot sequence number parameter determines which keys are visible. Any
 // keys not visible at the provided snapshot are ignored.
@@ -92,16 +92,20 @@ func (ui *UserIteratorConfig) Init(
 	lower, upper []byte,
 	hasPrefix *bool,
 	prefix *[]byte,
-	onlySets bool,
+	internalKeys bool,
 	bufs *Buffers,
 	iters ...keyspan.FragmentIterator,
 ) keyspan.FragmentIterator {
 	ui.snapshot = snapshot
 	ui.comparer = comparer
-	ui.onlySets = onlySets
+	ui.internalKeys = internalKeys
 	ui.miter.Init(comparer.Compare, ui, &bufs.merging, iters...)
 	ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix)
-	ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	if internalKeys {
+		ui.diter.Init(comparer, &ui.biter, keyspan.DefragmentInternal, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	} else {
+		ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	}
 	ui.litersUsed = 0
 	ui.bufs = bufs
 	return &ui.diter
@@ -137,7 +141,8 @@ func (ui *UserIteratorConfig) SetBounds(lower, upper []byte) {
 // of unset keys, removal of keys overwritten by a set at the same suffix, etc)
 // and then non-RangeKeySet keys are removed. The resulting transformed spans
 // only contain RangeKeySets describing the state visible at the provided
-// sequence number, and hold their Keys sorted by Suffix.
+// sequence number, and hold their Keys sorted by Suffix (except if internalKeys
+// is true, then keys remain sorted by trailer.
 func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error {
 	// Apply shadowing of keys.
 	dst.Start = s.Start
@@ -149,9 +154,16 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 	if err := coalesce(ui.comparer.Equal, &ui.bufs.sortBuf, ui.snapshot, s.Keys); err != nil {
 		return err
 	}
-	// During user iteration over range keys, unsets and deletes don't matter.
+	if ui.internalKeys {
-	// Remove them if onlySets = true. This step helps logical defragmentation
+		if s.KeysOrder != keyspan.ByTrailerDesc {
-	// during iteration.
+			panic("unexpected key ordering in UserIteratorTransform with internalKeys = true")
+		}
+		dst.Keys = ui.bufs.sortBuf.Keys
+		keyspan.SortKeysByTrailer(&dst.Keys)
+		return nil
+	}
+	// During user iteration over range keys, unsets and deletes don't matter. This
+	// step helps logical defragmentation during iteration.
 	keys := ui.bufs.sortBuf.Keys
 	dst.Keys = dst.Keys[:0]
 	for i := range keys {
@@ -165,17 +177,11 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
 				panic("pebble: keys unexpectedly not in ascending suffix order")
 			}
-			if ui.onlySets {
+			// Skip.
-				// Skip.
+			continue
-				continue
-			}
-			dst.Keys = append(dst.Keys, keys[i])
 		case base.InternalKeyKindRangeKeyDelete:
-			if ui.onlySets {
+			// Skip.
-				// Skip.
+			continue
-				continue
-			}
-			dst.Keys = append(dst.Keys, keys[i])
 		default:
 			return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
 		}
@@ -194,6 +200,10 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 // sequence numbers). It's intended for use during user iteration, when the
 // wrapped keyspan iterator is merging spans across all levels of the LSM.
 func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.Span) bool {
+	// This method is not called with internalKeys = true.
+	if ui.internalKeys {
+		panic("unexpected call to ShouldDefragment with internalKeys = true")
+	}
 	// This implementation must only be used on spans that have transformed by
 	// ui.Transform. The transform applies shadowing, removes all keys besides
 	// the resulting Sets and sorts the keys by suffix. Since shadowing has been
@@ -209,8 +219,8 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
 	ret := true
 	for i := range a.Keys {
 		if invariants.Enabled {
-			if ui.onlySets && (a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
+			if a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
-				b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet) {
+				b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet {
 				panic("pebble: unexpected non-RangeKeySet during defragmentation")
 			}
 			if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 ||
@@ -368,78 +378,3 @@ func coalesce(
 	}
 	return nil
 }
-
-// ForeignSSTTransformer implements a keyspan.Transformer for range keys in
-// foreign sstables (i.e. shared sstables not created by us). It is largely
-// similar to the Transform function implemented in UserIteratorConfig in that
-// it calls coalesce to remove range keys shadowed by other range keys, but also
-// retains the range key that does the shadowing. In addition, it elides
-// RangeKey unsets/dels in L6 as they are inapplicable when reading from a
-// different Pebble instance.
-type ForeignSSTTransformer struct {
-	Comparer *base.Comparer
-	Level    int
-	sortBuf  keyspan.KeysBySuffix
-}
-
-// Transform implements the Transformer interface.
-func (f *ForeignSSTTransformer) Transform(
-	cmp base.Compare, s keyspan.Span, dst *keyspan.Span,
-) error {
-	// Apply shadowing of keys.
-	dst.Start = s.Start
-	dst.End = s.End
-	f.sortBuf = keyspan.KeysBySuffix{
-		Cmp:  cmp,
-		Keys: f.sortBuf.Keys[:0],
-	}
-	if err := coalesce(f.Comparer.Equal, &f.sortBuf, math.MaxUint64, s.Keys); err != nil {
-		return err
-	}
-	keys := f.sortBuf.Keys
-	dst.Keys = dst.Keys[:0]
-	for i := range keys {
-		seqNum := keys[i].SeqNum()
-		switch keys[i].Kind() {
-		case base.InternalKeyKindRangeKeySet:
-			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
-				panic("pebble: keys unexpectedly not in ascending suffix order")
-			}
-			switch f.Level {
-			case 5:
-				fallthrough
-			case 6:
-				if seqNum != base.SeqNumForLevel(f.Level) {
-					panic(fmt.Sprintf("pebble: expected range key iter to return seqnum %d, got %d", base.SeqNumForLevel(f.Level), seqNum))
-				}
-			}
-		case base.InternalKeyKindRangeKeyUnset:
-			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
-				panic("pebble: keys unexpectedly not in ascending suffix order")
-			}
-			fallthrough
-		case base.InternalKeyKindRangeKeyDelete:
-			switch f.Level {
-			case 5:
-				// Emit this key.
-				if seqNum != base.SeqNumForLevel(f.Level) {
-					panic(fmt.Sprintf("pebble: expected range key iter to return seqnum %d, got %d", base.SeqNumForLevel(f.Level), seqNum))
-				}
-			case 6:
-				// Skip this key, as foreign sstable in L6 do not need to emit range key
-				// unsets/dels as they do not apply to any other sstables.
-				continue
-			}
-		default:
-			return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
-		}
-		dst.Keys = append(dst.Keys, keyspan.Key{
-			Trailer: base.MakeTrailer(seqNum, keys[i].Kind()),
-			Suffix:  keys[i].Suffix,
-			Value:   keys[i].Value,
-		})
-	}
-	// coalesce results in dst.Keys being sorted by Suffix.
-	dst.KeysOrder = keyspan.BySuffixAsc
-	return nil
-}

vendor/github.com/cockroachdb/pebble/internal/rate/rate.go

@@ -1,387 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package rate provides a rate limiter.
-package rate // import "github.com/cockroachdb/pebble/internal/rate"
-
-import (
-	"context"
-	"math"
-	"sync"
-	"time"
-
-	"github.com/cockroachdb/errors"
-)
-
-// Limit defines the maximum frequency of some events.
-// Limit is represented as number of events per second.
-// A zero Limit allows no events.
-type Limit float64
-
-// Inf is the infinite rate limit; it allows all events (even if burst is zero).
-const Inf = Limit(math.MaxFloat64)
-
-// Every converts a minimum time interval between events to a Limit.
-func Every(interval time.Duration) Limit {
-	if interval <= 0 {
-		return Inf
-	}
-	return 1 / Limit(interval.Seconds())
-}
-
-// A Limiter controls how frequently events are allowed to happen.
-// It implements a "token bucket" of size b, initially full and refilled
-// at rate r tokens per second.
-// Informally, in any large enough time interval, the Limiter limits the
-// rate to r tokens per second, with a maximum burst size of b events.
-// As a special case, if r == Inf (the infinite rate), b is ignored.
-// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets.
-//
-// The zero value is a valid Limiter, but it will reject all events.
-// Use NewLimiter to create non-zero Limiters.
-//
-// Limiter has three main methods, Allow, Reserve, and Wait.
-// Most callers should use Wait.
-//
-// Each of the three methods consumes a single token.
-// They differ in their behavior when no token is available.
-// If no token is available, Allow returns false.
-// If no token is available, Reserve returns a reservation for a future token
-// and the amount of time the caller must wait before using it.
-// If no token is available, Wait blocks until one can be obtained
-// or its associated context.Context is canceled.
-//
-// The methods AllowN, ReserveN, and WaitN consume n tokens.
-type Limiter struct {
-	limit Limit
-	burst int
-
-	mu     sync.Mutex
-	tokens float64
-	// last is the last time the limiter's tokens field was updated
-	last time.Time
-	// lastEvent is the latest time of a rate-limited event (past or future)
-	lastEvent time.Time
-}
-
-// Limit returns the maximum overall event rate.
-func (lim *Limiter) Limit() Limit {
-	lim.mu.Lock()
-	defer lim.mu.Unlock()
-	return lim.limit
-}
-
-// Burst returns the maximum burst size. Burst is the maximum number of tokens
-// that can be consumed in a single call to Allow, Reserve, or Wait, so higher
-// Burst values allow more events to happen at once.
-// A zero Burst allows no events, unless limit == Inf.
-func (lim *Limiter) Burst() int {
-	return lim.burst
-}
-
-// NewLimiter returns a new Limiter that allows events up to rate r and permits
-// bursts of at most b tokens.
-func NewLimiter(r Limit, b int) *Limiter {
-	return &Limiter{
-		limit: r,
-		burst: b,
-	}
-}
-
-// Allow is shorthand for AllowN(time.Now(), 1).
-func (lim *Limiter) Allow() bool {
-	return lim.AllowN(time.Now(), 1)
-}
-
-// AllowN reports whether n events may happen at time now.
-// Use this method if you intend to drop / skip events that exceed the rate limit.
-// Otherwise use Reserve or Wait.
-func (lim *Limiter) AllowN(now time.Time, n int) bool {
-	return lim.reserveN(now, n, 0).ok
-}
-
-// A Reservation holds information about events that are permitted by a Limiter to happen after a delay.
-// A Reservation may be canceled, which may enable the Limiter to permit additional events.
-type Reservation struct {
-	ok        bool
-	lim       *Limiter
-	tokens    int
-	timeToAct time.Time
-	// This is the Limit at reservation time, it can change later.
-	limit Limit
-}
-
-// OK returns whether the limiter can provide the requested number of tokens
-// within the maximum wait time.  If OK is false, Delay returns InfDuration, and
-// Cancel does nothing.
-func (r *Reservation) OK() bool {
-	return r.ok
-}
-
-// Delay is shorthand for DelayFrom(time.Now()).
-func (r *Reservation) Delay() time.Duration {
-	return r.DelayFrom(time.Now())
-}
-
-// InfDuration is the duration returned by Delay when a Reservation is not OK.
-const InfDuration = time.Duration(1<<63 - 1)
-
-// DelayFrom returns the duration for which the reservation holder must wait
-// before taking the reserved action.  Zero duration means act immediately.
-// InfDuration means the limiter cannot grant the tokens requested in this
-// Reservation within the maximum wait time.
-func (r *Reservation) DelayFrom(now time.Time) time.Duration {
-	if !r.ok {
-		return InfDuration
-	}
-	delay := r.timeToAct.Sub(now)
-	if delay < 0 {
-		return 0
-	}
-	return delay
-}
-
-// Cancel is shorthand for CancelAt(time.Now()).
-func (r *Reservation) Cancel() {
-	r.CancelAt(time.Now())
-}
-
-// CancelAt indicates that the reservation holder will not perform the reserved action
-// and reverses the effects of this Reservation on the rate limit as much as possible,
-// considering that other reservations may have already been made.
-func (r *Reservation) CancelAt(now time.Time) {
-	if !r.ok {
-		return
-	}
-
-	r.lim.mu.Lock()
-	defer r.lim.mu.Unlock()
-
-	if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(now) {
-		return
-	}
-
-	// calculate tokens to restore
-	// The duration between lim.lastEvent and r.timeToAct tells us how many tokens were reserved
-	// after r was obtained. These tokens should not be restored.
-	restoreTokens := float64(r.tokens) - r.limit.tokensFromDuration(r.lim.lastEvent.Sub(r.timeToAct))
-	if restoreTokens <= 0 {
-		return
-	}
-	// advance time to now
-	now, _, tokens := r.lim.advance(now)
-	// calculate new number of tokens
-	tokens += restoreTokens
-	if burst := float64(r.lim.burst); tokens > burst {
-		tokens = burst
-	}
-	// update state
-	r.lim.last = now
-	r.lim.tokens = tokens
-	if r.timeToAct == r.lim.lastEvent {
-		prevEvent := r.timeToAct.Add(r.limit.durationFromTokens(float64(-r.tokens)))
-		if !prevEvent.Before(now) {
-			r.lim.lastEvent = prevEvent
-		}
-	}
-}
-
-// Reserve is shorthand for ReserveN(time.Now(), 1).
-func (lim *Limiter) Reserve() Reservation {
-	return lim.ReserveN(time.Now(), 1)
-}
-
-// ReserveN returns a Reservation that indicates how long the caller must wait before n events happen.
-// The Limiter takes this Reservation into account when allowing future events.
-// ReserveN returns false if n exceeds the Limiter's burst size.
-// Usage example:
-//
-//	r := lim.ReserveN(time.Now(), 1)
-//	if !r.OK() {
-//	  // Not allowed to act! Did you remember to set lim.burst to be > 0 ?
-//	  return
-//	}
-//	time.Sleep(r.Delay())
-//	Act()
-//
-// Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events.
-// If you need to respect a deadline or cancel the delay, use Wait instead.
-// To drop or skip events exceeding rate limit, use Allow instead.
-func (lim *Limiter) ReserveN(now time.Time, n int) Reservation {
-	return lim.reserveN(now, n, InfDuration)
-}
-
-// Wait is shorthand for WaitN(ctx, 1).
-func (lim *Limiter) Wait(ctx context.Context) (err error) {
-	return lim.WaitN(ctx, 1)
-}
-
-// WaitN blocks until lim permits n events to happen.
-// It returns an error if n exceeds the Limiter's burst size, the Context is
-// canceled, or the expected wait time exceeds the Context's Deadline.
-func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
-	if n > lim.burst && lim.limit != Inf {
-		return errors.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", errors.Safe(n), errors.Safe(lim.burst))
-	}
-	// Check if ctx is already cancelled
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	default:
-	}
-	// Determine wait limit
-	now := time.Now()
-	waitLimit := InfDuration
-	if deadline, ok := ctx.Deadline(); ok {
-		waitLimit = deadline.Sub(now)
-	}
-	// Reserve
-	r := lim.reserveN(now, n, waitLimit)
-	if !r.ok {
-		return errors.Errorf("rate: Wait(n=%d) would exceed context deadline", errors.Safe(n))
-	}
-	// Wait
-	d := r.DelayFrom(now)
-	if d <= 0 {
-		return nil
-	}
-	t := time.NewTimer(d)
-	defer t.Stop()
-	select {
-	case <-t.C:
-		// We can proceed.
-		return nil
-	case <-ctx.Done():
-		// Context was canceled before we could proceed.  Cancel the
-		// reservation, which may permit other events to proceed sooner.
-		r.Cancel()
-		return ctx.Err()
-	}
-}
-
-// Delay is shorthand for DelayN(time.Now(), 1).
-func (lim *Limiter) Delay() time.Duration {
-	return lim.DelayN(time.Now(), 1)
-}
-
-// DelayN returns the delay to wait to permit n events to happen. Zero duration
-// means act immediately. InfDuration means the limiter cannot grant the tokens
-// requested within the maximum wait time.
-func (lim *Limiter) DelayN(now time.Time, n int) time.Duration {
-	r := lim.reserveN(now, n, InfDuration)
-	return r.DelayFrom(now)
-}
-
-// SetLimit is shorthand for SetLimitAt(time.Now(), newLimit).
-func (lim *Limiter) SetLimit(newLimit Limit) {
-	lim.SetLimitAt(time.Now(), newLimit)
-}
-
-// SetLimitAt sets a new Limit for the limiter. The new Limit, and Burst, may be violated
-// or underutilized by those which reserved (using Reserve or Wait) but did not yet act
-// before SetLimitAt was called.
-func (lim *Limiter) SetLimitAt(now time.Time, newLimit Limit) {
-	lim.mu.Lock()
-	defer lim.mu.Unlock()
-
-	now, _, tokens := lim.advance(now)
-
-	lim.last = now
-	lim.tokens = tokens
-	lim.limit = newLimit
-}
-
-// reserveN is a helper method for AllowN, ReserveN, and WaitN.
-// maxFutureReserve specifies the maximum reservation wait duration allowed.
-// reserveN returns Reservation, not *Reservation, to avoid allocation in AllowN and WaitN.
-func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duration) Reservation {
-	lim.mu.Lock()
-
-	if lim.limit == Inf {
-		lim.mu.Unlock()
-		return Reservation{
-			ok:        true,
-			lim:       lim,
-			tokens:    n,
-			timeToAct: now,
-		}
-	}
-
-	now, last, tokens := lim.advance(now)
-
-	// Calculate the remaining number of tokens resulting from the request.
-	tokens -= float64(n)
-
-	// Calculate the wait duration
-	var waitDuration time.Duration
-	if tokens < 0 {
-		waitDuration = lim.limit.durationFromTokens(-tokens)
-	}
-
-	// Decide result
-	ok := n <= lim.burst && waitDuration <= maxFutureReserve
-
-	// Prepare reservation
-	r := Reservation{
-		ok:    ok,
-		lim:   lim,
-		limit: lim.limit,
-	}
-	if ok {
-		r.tokens = n
-		r.timeToAct = now.Add(waitDuration)
-	}
-
-	// Update state
-	if ok {
-		lim.last = now
-		lim.tokens = tokens
-		lim.lastEvent = r.timeToAct
-	} else {
-		lim.last = last
-	}
-
-	lim.mu.Unlock()
-	return r
-}
-
-// advance calculates and returns an updated state for lim resulting from the passage of time.
-// lim is not changed.
-func (lim *Limiter) advance(
-	now time.Time,
-) (newNow time.Time, newLast time.Time, newTokens float64) {
-	last := lim.last
-	if now.Before(last) {
-		last = now
-	}
-
-	// Avoid making delta overflow below when last is very old.
-	maxElapsed := lim.limit.durationFromTokens(float64(lim.burst) - lim.tokens)
-	elapsed := now.Sub(last)
-	if elapsed > maxElapsed {
-		elapsed = maxElapsed
-	}
-
-	// Calculate the new number of tokens, due to time that passed.
-	delta := lim.limit.tokensFromDuration(elapsed)
-	tokens := lim.tokens + delta
-	if burst := float64(lim.burst); tokens > burst {
-		tokens = burst
-	}
-
-	return now, last, tokens
-}
-
-// durationFromTokens is a unit conversion function from the number of tokens to the duration
-// of time it takes to accumulate them at a rate of limit tokens per second.
-func (limit Limit) durationFromTokens(tokens float64) time.Duration {
-	seconds := tokens / float64(limit)
-	return time.Nanosecond * time.Duration(1e9*seconds)
-}
-
-// tokensFromDuration is a unit conversion function from a time duration to the number of tokens
-// which could be accumulated during that duration at a rate of limit tokens per second.
-func (limit Limit) tokensFromDuration(d time.Duration) float64 {
-	return d.Seconds() * float64(limit)
-}

vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go

@@ -20,17 +20,27 @@ import (
 	"strings"
 
 	"github.com/cockroachdb/pebble/internal/base"
+	"golang.org/x/exp/constraints"
+	"golang.org/x/exp/rand"
 )
 
 const alpha = "abcdefghijklmnopqrstuvwxyz"
 
 const suffixDelim = '@'
 
+var inverseAlphabet = make(map[byte]int64, len(alpha))
+
+func init() {
+	for i := range alpha {
+		inverseAlphabet[alpha[i]] = int64(i)
+	}
+}
+
 // MaxSuffixLen is the maximum length of a suffix generated by this package.
-var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", math.MaxInt64))
+var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", int64(math.MaxInt64)))
 
 // Comparer is the comparer for test keys generated by this package.
-var Comparer *base.Comparer = &base.Comparer{
+var Comparer = &base.Comparer{
 	Compare: compare,
 	Equal:   func(a, b []byte) bool { return compare(a, b) == 0 },
 	AbbreviatedKey: func(k []byte) uint64 {
@@ -136,7 +146,7 @@ func compareTimestamps(a, b []byte) int {
 // Keyspace describes a finite keyspace of unsuffixed test keys.
 type Keyspace interface {
 	// Count returns the number of keys that exist within this keyspace.
-	Count() int
+	Count() int64
 
 	// MaxLen returns the maximum length, in bytes, of a key within this
 	// keyspace. This is only guaranteed to return an upper bound.
@@ -144,20 +154,21 @@ type Keyspace interface {
 
 	// Slice returns the sub-keyspace from index i, inclusive, to index j,
 	// exclusive. The receiver is unmodified.
-	Slice(i, j int) Keyspace
+	Slice(i, j int64) Keyspace
 
 	// EveryN returns a key space that includes 1 key for every N keys in the
 	// original keyspace. The receiver is unmodified.
-	EveryN(n int) Keyspace
+	EveryN(n int64) Keyspace
 
-	key(buf []byte, i int) int
+	// key writes the i-th key to the buffer and returns the length.
+	key(buf []byte, i int64) int
 }
 
 // Divvy divides the provided keyspace into N equal portions, containing
 // disjoint keys evenly distributed across the keyspace.
-func Divvy(ks Keyspace, n int) []Keyspace {
+func Divvy(ks Keyspace, n int64) []Keyspace {
 	ret := make([]Keyspace, n)
-	for i := 0; i < n; i++ {
+	for i := int64(0); i < n; i++ {
 		ret[i] = ks.Slice(i, ks.Count()).EveryN(n)
 	}
 	return ret
@@ -175,7 +186,7 @@ func Alpha(maxLength int) Keyspace {
 
 // KeyAt returns the i-th key within the keyspace with a suffix encoding the
 // timestamp t.
-func KeyAt(k Keyspace, i int, t int) []byte {
+func KeyAt(k Keyspace, i int64, t int64) []byte {
 	b := make([]byte, k.MaxLen()+MaxSuffixLen)
 	return b[:WriteKeyAt(b, k, i, t)]
 }
@@ -183,20 +194,20 @@ func KeyAt(k Keyspace, i int, t int) []byte {
 // WriteKeyAt writes the i-th key within the keyspace to the buffer dst, with a
 // suffix encoding the timestamp t suffix. It returns the number of bytes
 // written.
-func WriteKeyAt(dst []byte, k Keyspace, i int, t int) int {
+func WriteKeyAt(dst []byte, k Keyspace, i int64, t int64) int {
 	n := WriteKey(dst, k, i)
 	n += WriteSuffix(dst[n:], t)
 	return n
 }
 
 // Suffix returns the test keys suffix representation of timestamp t.
-func Suffix(t int) []byte {
+func Suffix(t int64) []byte {
 	b := make([]byte, MaxSuffixLen)
 	return b[:WriteSuffix(b, t)]
 }
 
 // SuffixLen returns the exact length of the given suffix when encoded.
-func SuffixLen(t int) int {
+func SuffixLen(t int64) int {
 	// Begin at 1 for the '@' delimiter, 1 for a single digit.
 	n := 2
 	t /= 10
@@ -208,42 +219,42 @@ func SuffixLen(t int) int {
 }
 
 // ParseSuffix returns the integer representation of the encoded suffix.
-func ParseSuffix(s []byte) (int, error) {
+func ParseSuffix(s []byte) (int64, error) {
-	return strconv.Atoi(strings.TrimPrefix(string(s), string(suffixDelim)))
+	return strconv.ParseInt(strings.TrimPrefix(string(s), string(suffixDelim)), 10, 64)
 }
 
 // WriteSuffix writes the test keys suffix representation of timestamp t to dst,
 // returning the number of bytes written.
-func WriteSuffix(dst []byte, t int) int {
+func WriteSuffix(dst []byte, t int64) int {
 	dst[0] = suffixDelim
 	n := 1
-	n += len(strconv.AppendInt(dst[n:n], int64(t), 10))
+	n += len(strconv.AppendInt(dst[n:n], t, 10))
 	return n
 }
 
 // Key returns the i-th unsuffixed key within the keyspace.
-func Key(k Keyspace, i int) []byte {
+func Key(k Keyspace, i int64) []byte {
 	b := make([]byte, k.MaxLen())
 	return b[:k.key(b, i)]
 }
 
 // WriteKey writes the i-th unsuffixed key within the keyspace to the buffer dst. It
 // returns the number of bytes written.
-func WriteKey(dst []byte, k Keyspace, i int) int {
+func WriteKey(dst []byte, k Keyspace, i int64) int {
 	return k.key(dst, i)
 }
 
 type alphabet struct {
 	alphabet  []byte
 	maxLength int
-	headSkip  int
+	headSkip  int64
-	tailSkip  int
+	tailSkip  int64
-	increment int
+	increment int64
 }
 
-func (a alphabet) Count() int {
+func (a alphabet) Count() int64 {
 	// Calculate the total number of keys, ignoring the increment.
-	total := (keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip)
+	total := keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip
 
 	// The increment dictates that we take every N keys, where N = a.increment.
 	// Consider a total containing the 5 keys:
@@ -263,20 +274,25 @@ func (a alphabet) MaxLen() int {
 	return a.maxLength
 }
 
-func (a alphabet) Slice(i, j int) Keyspace {
+func (a alphabet) Slice(i, j int64) Keyspace {
 	s := a
 	s.headSkip += i
 	s.tailSkip += a.Count() - j
 	return s
 }
 
-func (a alphabet) EveryN(n int) Keyspace {
+func (a alphabet) EveryN(n int64) Keyspace {
 	s := a
 	s.increment *= n
 	return s
 }
 
-func keyCount(n, l int) int {
+func keyCount(n, l int) int64 {
+	if n == 0 {
+		return 0
+	} else if n == 1 {
+		return int64(l)
+	}
 	// The number of representable keys in the keyspace is a function of the
 	// length of the alphabet n and the max key length l. Consider how the
 	// number of representable keys grows as l increases:
@@ -286,10 +302,10 @@ func keyCount(n, l int) int {
 	// l = 3: n + n^2 + n^3
 	// ...
 	// Σ i=(1...l) n^i = n*(n^l - 1)/(n-1)
-	return (n * (int(math.Pow(float64(n), float64(l))) - 1)) / (n - 1)
+	return (int64(n) * (int64(math.Pow(float64(n), float64(l))) - 1)) / int64(n-1)
 }
 
-func (a alphabet) key(buf []byte, idx int) int {
+func (a alphabet) key(buf []byte, idx int64) int {
 	// This function generates keys of length 1..maxKeyLength, pulling
 	// characters from the alphabet. The idx determines which key to generate,
 	// generating the i-th lexicographically next key.
@@ -308,14 +324,14 @@ func (a alphabet) key(buf []byte, idx int) int {
 		keyCount(len(a.alphabet), a.maxLength))
 }
 
-func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int {
+func generateAlphabetKey(buf, alphabet []byte, i, keyCount int64) int {
 	if keyCount == 0 || i > keyCount || i < 0 {
 		return 0
 	}
 
 	// Of the keyCount keys in the generative keyspace, how many are there
 	// starting with a particular character?
-	keysPerCharacter := keyCount / len(alphabet)
+	keysPerCharacter := keyCount / int64(len(alphabet))
 
 	// Find the character that the key at index i starts with and set it.
 	characterIdx := i / keysPerCharacter
@@ -347,3 +363,156 @@ func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int {
 	i = i - keysPerCharacter*characterIdx - 1
 	return 1 + generateAlphabetKey(buf[1:], alphabet, i, keysPerCharacter-1)
 }
+
+// computeAlphabetKeyIndex computes the inverse of generateAlphabetKey,
+// returning the index of a particular key, given the provided alphabet and max
+// length of a key.
+//
+// len(key) must be ≥ 1.
+func computeAlphabetKeyIndex(key []byte, alphabet map[byte]int64, n int) int64 {
+	i, ok := alphabet[key[0]]
+	if !ok {
+		panic(fmt.Sprintf("unrecognized alphabet character %v", key[0]))
+	}
+	// How many keys exist that start with the preceding i characters? Each of
+	// the i characters themselves are a key, plus the count of all the keys
+	// with one less character for each.
+	ret := i + i*keyCount(len(alphabet), n-1)
+	if len(key) > 1 {
+		ret += 1 + computeAlphabetKeyIndex(key[1:], alphabet, n-1)
+	}
+	return ret
+}
+
+func abs(a int64) int64 {
+	if a < 0 {
+		return -a
+	}
+	return a
+}
+
+// RandomSeparator returns a random alphabetic key k such that a < k < b,
+// pulling randomness from the provided random number generator. If dst is
+// provided and the generated key fits within dst's capacity, the returned slice
+// will use dst's memory.
+//
+// If a prefix P exists such that Prefix(a) < P < Prefix(b), the generated key
+// will consist of the prefix P appended with the provided suffix. A zero suffix
+// generates an unsuffixed key. If no such prefix P exists, RandomSeparator will
+// try to find a key k with either Prefix(a) or Prefix(b) such that a < k < b,
+// but the generated key will not use the provided suffix. Note that it's
+// possible that no separator key exists (eg, a='a@2', b='a@1'), in which case
+// RandomSeparator returns nil.
+//
+// If RandomSeparator generates a new prefix, the generated prefix will have
+// length at most MAX(maxLength, len(Prefix(a)), len(Prefix(b))).
+//
+// RandomSeparator panics if a or b fails to decode.
+func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Rand) []byte {
+	if Comparer.Compare(a, b) >= 0 {
+		return nil
+	}
+
+	// Determine both keys' logical prefixes and suffixes.
+	ai := Comparer.Split(a)
+	bi := Comparer.Split(b)
+	ap := a[:ai]
+	bp := b[:bi]
+	maxLength = max[int](maxLength, max[int](len(ap), len(bp)))
+	var as, bs int64
+	var err error
+	if ai != len(a) {
+		as, err = ParseSuffix(a[ai:])
+		if err != nil {
+			panic(fmt.Sprintf("failed to parse suffix of %q", a))
+		}
+	}
+	if bi != len(b) {
+		bs, err = ParseSuffix(b[bi:])
+		if err != nil {
+			panic(fmt.Sprintf("failed to parse suffix of %q", b))
+		}
+	}
+
+	apIdx := computeAlphabetKeyIndex(ap, inverseAlphabet, maxLength)
+	bpIdx := computeAlphabetKeyIndex(bp, inverseAlphabet, maxLength)
+	diff := bpIdx - apIdx
+	generatedIdx := bpIdx
+	if diff > 0 {
+		var add int64 = diff + 1
+		var start int64 = apIdx
+		if as == 1 {
+			// There's no expressible key with prefix a greater than a@1. So,
+			// exclude ap.
+			start = apIdx + 1
+			add = diff
+		}
+		if bs == 0 {
+			// No key with prefix b can sort before b@0. We don't want to pick b.
+			add--
+		}
+		// We're allowing generated id to be in the range [start, start + add - 1].
+		if start > start+add-1 {
+			return nil
+		}
+		// If we can generate a key which is actually in the middle of apIdx
+		// and bpIdx use it so that we don't have to bother about timestamps.
+		generatedIdx = rng.Int63n(add) + start
+		for diff > 1 && generatedIdx == apIdx || generatedIdx == bpIdx {
+			generatedIdx = rng.Int63n(add) + start
+		}
+	}
+
+	switch {
+	case generatedIdx == apIdx && generatedIdx == bpIdx:
+		if abs(bs-as) <= 1 {
+			// There's no expressible suffix between the two, and there's no
+			// possible separator key.
+			return nil
+		}
+		// The key b is >= key a, but has the same prefix, so b must have the
+		// smaller timestamp, unless a has timestamp of 0.
+		//
+		// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
+		// any suffix we generate will be greater than it.
+		if as == 0 {
+			// bs > as
+			suffix = bs + rng.Int63n(10) + 1
+		} else {
+			// bs < as.
+			// Generate suffix in range [bs + 1, as - 1]
+			suffix = bs + 1 + rng.Int63n(as-bs-1)
+		}
+	case generatedIdx == apIdx:
+		// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
+		// any suffix we generate will be greater than it.
+		if as == 0 && suffix == 0 {
+			suffix++
+		} else if as != 0 && suffix >= as {
+			suffix = rng.Int63n(as)
+		}
+	case generatedIdx == bpIdx:
+		if suffix <= bs {
+			suffix = bs + rng.Int63n(10) + 1
+		}
+	}
+	if sz := maxLength + SuffixLen(suffix); cap(dst) < sz {
+		dst = make([]byte, sz)
+	} else {
+		dst = dst[:cap(dst)]
+	}
+	var w int
+	if suffix == 0 {
+		w = WriteKey(dst, Alpha(maxLength), generatedIdx)
+	} else {
+		w = WriteKeyAt(dst, Alpha(maxLength), generatedIdx, suffix)
+	}
+	return dst[:w]
+}
+
+func max[I constraints.Ordered](a, b I) I {
+	if b > a {
+		return b
+	}
+	return a
+}

vendor/github.com/cockroachdb/pebble/iterator.go

@@ -154,6 +154,18 @@ func (s *RangeKeyIteratorStats) Merge(o RangeKeyIteratorStats) {
 	s.SkippedPoints += o.SkippedPoints
 }
 
+func (s *RangeKeyIteratorStats) String() string {
+	return redact.StringWithoutMarkers(s)
+}
+
+// SafeFormat implements the redact.SafeFormatter interface.
+func (s *RangeKeyIteratorStats) SafeFormat(p redact.SafePrinter, verb rune) {
+	p.Printf("range keys: %s, contained points: %s (%s skipped)",
+		humanize.Count.Uint64(uint64(s.Count)),
+		humanize.Count.Uint64(uint64(s.ContainedPoints)),
+		humanize.Count.Uint64(uint64(s.SkippedPoints)))
+}
+
 // LazyValue is a lazy value. See the long comment in base.LazyValue.
 type LazyValue = base.LazyValue
 
@@ -187,7 +199,9 @@ type Iterator struct {
 	comparer  base.Comparer
 	iter      internalIterator
 	pointIter internalIterator
+	// Either readState or version is set, but not both.
 	readState *readState
+	version   *version
 	// rangeKey holds iteration state specific to iteration over range keys.
 	// The range key field may be nil if the Iterator has never been configured
 	// to iterate over range keys. Its non-nilness cannot be used to determine
@@ -445,6 +459,11 @@ const (
 	seekPrefixGELastPositioningOp
 	seekGELastPositioningOp
 	seekLTLastPositioningOp
+	// internalNextOp is a special internal iterator positioning operation used
+	// by CanDeterministicallySingleDelete. It exists for enforcing requirements
+	// around calling CanDeterministicallySingleDelete at most once per external
+	// iterator position.
+	internalNextOp
 	// invalidatedLastPositionOp is similar to unknownLastPositionOp and the
 	// only reason to distinguish this is for the wider set of SeekGE
 	// optimizations we permit for the external iterator Iterator.forwardOnly
@@ -548,6 +567,21 @@ func (i *Iterator) findNextEntry(limit []byte) {
 			return
 		}
 
+		// If the user has configured a SkipPoint function, invoke it to see
+		// whether we should skip over the current user key.
+		if i.opts.SkipPoint != nil && key.Kind() != InternalKeyKindRangeKeySet && i.opts.SkipPoint(i.iterKey.UserKey) {
+			// NB: We could call nextUserKey, but in some cases the SkipPoint
+			// predicate function might be cheaper than nextUserKey's key copy
+			// and key comparison. This should be the case for MVCC suffix
+			// comparisons, for example. In the future, we could expand the
+			// SkipPoint interface to give the implementor more control over
+			// whether we skip over just the internal key, the user key, or even
+			// the key prefix.
+			i.stats.ForwardStepCount[InternalIterCall]++
+			i.iterKey, i.iterValue = i.iter.Next()
+			continue
+		}
+
 		switch key.Kind() {
 		case InternalKeyKindRangeKeySet:
 			// Save the current key.
@@ -617,6 +651,13 @@ func (i *Iterator) findNextEntry(limit []byte) {
 }
 
 func (i *Iterator) nextPointCurrentUserKey() bool {
+	// If the user has configured a SkipPoint function and the current user key
+	// would be skipped by it, there's no need to step forward looking for a
+	// point key. If we were to find one, it should be skipped anyways.
+	if i.opts.SkipPoint != nil && i.opts.SkipPoint(i.key) {
+		return false
+	}
+
 	i.pos = iterPosCurForward
 
 	i.iterKey, i.iterValue = i.iter.Next()
@@ -778,39 +819,41 @@ func (i *Iterator) maybeSampleRead() {
 func (i *Iterator) sampleRead() {
 	var topFile *manifest.FileMetadata
 	topLevel, numOverlappingLevels := numLevels, 0
-	if mi, ok := i.iter.(*mergingIter); ok {
+	mi := i.merging
-		if len(mi.levels) > 1 {
+	if mi == nil {
-			mi.ForEachLevelIter(func(li *levelIter) bool {
+		return
-				l := manifest.LevelToInt(li.level)
+	}
-				if f := li.iterFile; f != nil {
+	if len(mi.levels) > 1 {
-					var containsKey bool
+		mi.ForEachLevelIter(func(li *levelIter) bool {
-					if i.pos == iterPosNext || i.pos == iterPosCurForward ||
+			l := manifest.LevelToInt(li.level)
-						i.pos == iterPosCurForwardPaused {
+			if f := li.iterFile; f != nil {
-						containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0
+				var containsKey bool
-					} else if i.pos == iterPosPrev || i.pos == iterPosCurReverse ||
+				if i.pos == iterPosNext || i.pos == iterPosCurForward ||
-						i.pos == iterPosCurReversePaused {
+					i.pos == iterPosCurForwardPaused {
-						containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0
+					containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0
-					}
+				} else if i.pos == iterPosPrev || i.pos == iterPosCurReverse ||
-					// Do nothing if the current key is not contained in f's
+					i.pos == iterPosCurReversePaused {
-					// bounds. We could seek the LevelIterator at this level
+					containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0
-					// to find the right file, but the performance impacts of
-					// doing that are significant enough to negate the benefits
-					// of read sampling in the first place. See the discussion
-					// at:
-					// https://github.com/cockroachdb/pebble/pull/1041#issuecomment-763226492
-					if containsKey {
-						numOverlappingLevels++
-						if numOverlappingLevels >= 2 {
-							// Terminate the loop early if at least 2 overlapping levels are found.
-							return true
-						}
-						topLevel = l
-						topFile = f
-					}
 				}
-				return false
+				// Do nothing if the current key is not contained in f's
-			})
+				// bounds. We could seek the LevelIterator at this level
-		}
+				// to find the right file, but the performance impacts of
+				// doing that are significant enough to negate the benefits
+				// of read sampling in the first place. See the discussion
+				// at:
+				// https://github.com/cockroachdb/pebble/pull/1041#issuecomment-763226492
+				if containsKey {
+					numOverlappingLevels++
+					if numOverlappingLevels >= 2 {
+						// Terminate the loop early if at least 2 overlapping levels are found.
+						return true
+					}
+					topLevel = l
+					topFile = f
+				}
+			}
+			return false
+		})
 	}
 	if topFile == nil || topLevel >= numLevels {
 		return
@@ -907,6 +950,26 @@ func (i *Iterator) findPrevEntry(limit []byte) {
 			}
 		}
 
+		// If the user has configured a SkipPoint function, invoke it to see
+		// whether we should skip over the current user key.
+		if i.opts.SkipPoint != nil && key.Kind() != InternalKeyKindRangeKeySet && i.opts.SkipPoint(key.UserKey) {
+			// NB: We could call prevUserKey, but in some cases the SkipPoint
+			// predicate function might be cheaper than prevUserKey's key copy
+			// and key comparison. This should be the case for MVCC suffix
+			// comparisons, for example. In the future, we could expand the
+			// SkipPoint interface to give the implementor more control over
+			// whether we skip over just the internal key, the user key, or even
+			// the key prefix.
+			i.stats.ReverseStepCount[InternalIterCall]++
+			i.iterKey, i.iterValue = i.iter.Prev()
+			if limit != nil && i.iterKey != nil && i.cmp(limit, i.iterKey.UserKey) > 0 && !i.rangeKeyWithinLimit(limit) {
+				i.iterValidityState = IterAtLimit
+				i.pos = iterPosCurReversePaused
+				return
+			}
+			continue
+		}
+
 		switch key.Kind() {
 		case InternalKeyKindRangeKeySet:
 			// Range key start boundary markers are interleaved with the maximum
@@ -944,12 +1007,12 @@ func (i *Iterator) findPrevEntry(limit []byte) {
 			// Compare with the limit. We could optimize by only checking when
 			// we step to the previous user key, but detecting that requires a
 			// comparison too. Note that this position may already passed a
-			// number of versions of this user key, but they are all deleted,
+			// number of versions of this user key, but they are all deleted, so
-			// so the fact that a subsequent Prev*() call will not see them is
+			// the fact that a subsequent Prev*() call will not see them is
 			// harmless. Also note that this is the only place in the loop,
-			// other than the firstLoopIter case above, where we could step
+			// other than the firstLoopIter and SkipPoint cases above, where we
-			// to a different user key and start processing it for returning
+			// could step to a different user key and start processing it for
-			// to the caller.
+			// returning to the caller.
 			if limit != nil && i.iterKey != nil && i.cmp(limit, i.iterKey.UserKey) > 0 && !i.rangeKeyWithinLimit(limit) {
 				i.iterValidityState = IterAtLimit
 				i.pos = iterPosCurReversePaused
@@ -1760,7 +1823,8 @@ func (i *Iterator) internalNextPrefix(currKeyPrefixLen int) {
 	i.iterKey, i.iterValue = i.iter.NextPrefix(i.prefixOrFullSeekKey)
 	if invariants.Enabled && i.iterKey != nil {
 		if iterKeyPrefixLen := i.split(i.iterKey.UserKey); i.cmp(i.iterKey.UserKey[:iterKeyPrefixLen], i.prefixOrFullSeekKey) < 0 {
-			panic("pebble: iter.NextPrefix did not advance beyond the current prefix")
+			panic(errors.AssertionFailedf("pebble: iter.NextPrefix did not advance beyond the current prefix: now at %q; expected to be geq %q",
+				i.iterKey, i.prefixOrFullSeekKey))
 		}
 	}
 }
@@ -2129,6 +2193,9 @@ func (i *Iterator) Value() []byte {
 
 // ValueAndErr returns the value, and any error encountered in extracting the value.
 // REQUIRES: i.Error()==nil and HasPointAndRange() returns true for hasPoint.
+//
+// The caller should not modify the contents of the returned slice, and its
+// contents may change on the next call to Next.
 func (i *Iterator) ValueAndErr() ([]byte, error) {
 	val, callerOwned, err := i.value.Value(i.lazyValueBuf)
 	if err != nil {
@@ -2159,7 +2226,13 @@ func (i *Iterator) RangeKeys() []RangeKeyData {
 // Valid returns true if the iterator is positioned at a valid key/value pair
 // and false otherwise.
 func (i *Iterator) Valid() bool {
-	return i.iterValidityState == IterValid && !i.requiresReposition
+	valid := i.iterValidityState == IterValid && !i.requiresReposition
+	if invariants.Enabled {
+		if err := i.Error(); valid && err != nil {
+			panic(errors.WithSecondaryError(errors.AssertionFailedf("pebble: iterator is valid with non-nil Error"), err))
+		}
+	}
+	return valid
 }
 
 // Error returns any accumulated error.
@@ -2224,6 +2297,10 @@ func (i *Iterator) Close() error {
 		i.readState = nil
 	}
 
+	if i.version != nil {
+		i.version.Unref()
+	}
+
 	for _, readers := range i.externalReaders {
 		for _, r := range readers {
 			err = firstError(err, r.Close())
@@ -2416,7 +2493,8 @@ func (i *Iterator) SetOptions(o *IterOptions) {
 	// If either options specify block property filters for an iterator stack,
 	// reconstruct it.
 	if i.pointIter != nil && (closeBoth || len(o.PointKeyFilters) > 0 || len(i.opts.PointKeyFilters) > 0 ||
-		o.RangeKeyMasking.Filter != nil || i.opts.RangeKeyMasking.Filter != nil) {
+		o.RangeKeyMasking.Filter != nil || i.opts.RangeKeyMasking.Filter != nil || o.SkipPoint != nil ||
+		i.opts.SkipPoint != nil) {
 		i.err = firstError(i.err, i.pointIter.Close())
 		i.pointIter = nil
 	}
@@ -2655,11 +2733,22 @@ func (i *Iterator) CloneWithContext(ctx context.Context, opts CloneOptions) (*It
 	}
 
 	readState := i.readState
-	if readState == nil {
+	vers := i.version
+	if readState == nil && vers == nil {
 		return nil, errors.Errorf("cannot Clone a closed Iterator")
 	}
 	// i is already holding a ref, so there is no race with unref here.
-	readState.ref()
+	//
+	// TODO(bilal): If the underlying iterator was created on a snapshot, we could
+	// grab a reference to the current readState instead of reffing the original
+	// readState. This allows us to release references to some zombie sstables
+	// and memtables.
+	if readState != nil {
+		readState.ref()
+	}
+	if vers != nil {
+		vers.Ref()
+	}
 	// Bundle various structures under a single umbrella in order to allocate
 	// them together.
 	buf := iterAllocPool.Get().(*iterAlloc)
@@ -2671,6 +2760,7 @@ func (i *Iterator) CloneWithContext(ctx context.Context, opts CloneOptions) (*It
 		merge:               i.merge,
 		comparer:            i.comparer,
 		readState:           readState,
+		version:             vers,
 		keyBuf:              buf.keyBuf,
 		prefixOrFullSeekKey: buf.prefixOrFullSeekKey,
 		boundsBuf:           buf.boundsBuf,
@@ -2710,43 +2800,209 @@ func (stats *IteratorStats) String() string {
 
 // SafeFormat implements the redact.SafeFormatter interface.
 func (stats *IteratorStats) SafeFormat(s redact.SafePrinter, verb rune) {
-	for i := range stats.ForwardStepCount {
+	if stats.ReverseSeekCount[InterfaceCall] == 0 && stats.ReverseSeekCount[InternalIterCall] == 0 {
-		switch IteratorStatsKind(i) {
+		s.Printf("seeked %s times (%s internal)",
-		case InterfaceCall:
+			humanize.Count.Uint64(uint64(stats.ForwardSeekCount[InterfaceCall])),
-			s.SafeString("(interface (dir, seek, step): ")
+			humanize.Count.Uint64(uint64(stats.ForwardSeekCount[InternalIterCall])),
-		case InternalIterCall:
-			s.SafeString(", (internal (dir, seek, step): ")
-		}
-		s.Printf("(fwd, %d, %d), (rev, %d, %d))",
-			redact.Safe(stats.ForwardSeekCount[i]), redact.Safe(stats.ForwardStepCount[i]),
-			redact.Safe(stats.ReverseSeekCount[i]), redact.Safe(stats.ReverseStepCount[i]))
-	}
-	if stats.InternalStats != (InternalIteratorStats{}) {
-		s.SafeString(",\n(internal-stats: ")
-		s.Printf("(block-bytes: (total %s, cached %s, read-time %s)), "+
-			"(points: (count %s, key-bytes %s, value-bytes %s, tombstoned %s))",
-			humanize.IEC.Uint64(stats.InternalStats.BlockBytes),
-			humanize.IEC.Uint64(stats.InternalStats.BlockBytesInCache),
-			humanize.FormattedString(stats.InternalStats.BlockReadDuration.String()),
-			humanize.SI.Uint64(stats.InternalStats.PointCount),
-			humanize.SI.Uint64(stats.InternalStats.KeyBytes),
-			humanize.SI.Uint64(stats.InternalStats.ValueBytes),
-			humanize.SI.Uint64(stats.InternalStats.PointsCoveredByRangeTombstones),
 		)
-		if stats.InternalStats.SeparatedPointValue.Count != 0 {
+	} else {
-			s.Printf(", (separated: (count %s, bytes %s, fetched %s)))",
+		s.Printf("seeked %s times (%s fwd/%s rev, internal: %s fwd/%s rev)",
-				humanize.SI.Uint64(stats.InternalStats.SeparatedPointValue.Count),
+			humanize.Count.Uint64(uint64(stats.ForwardSeekCount[InterfaceCall]+stats.ReverseSeekCount[InterfaceCall])),
-				humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytes),
+			humanize.Count.Uint64(uint64(stats.ForwardSeekCount[InterfaceCall])),
-				humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytesFetched))
+			humanize.Count.Uint64(uint64(stats.ReverseSeekCount[InterfaceCall])),
-		} else {
+			humanize.Count.Uint64(uint64(stats.ForwardSeekCount[InternalIterCall])),
-			s.Printf(")")
+			humanize.Count.Uint64(uint64(stats.ReverseSeekCount[InternalIterCall])),
-		}
+		)
+	}
+	s.SafeString("; ")
+
+	if stats.ReverseStepCount[InterfaceCall] == 0 && stats.ReverseStepCount[InternalIterCall] == 0 {
+		s.Printf("stepped %s times (%s internal)",
+			humanize.Count.Uint64(uint64(stats.ForwardStepCount[InterfaceCall])),
+			humanize.Count.Uint64(uint64(stats.ForwardStepCount[InternalIterCall])),
+		)
+	} else {
+		s.Printf("stepped %s times (%s fwd/%s rev, internal: %s fwd/%s rev)",
+			humanize.Count.Uint64(uint64(stats.ForwardStepCount[InterfaceCall]+stats.ReverseStepCount[InterfaceCall])),
+			humanize.Count.Uint64(uint64(stats.ForwardStepCount[InterfaceCall])),
+			humanize.Count.Uint64(uint64(stats.ReverseStepCount[InterfaceCall])),
+			humanize.Count.Uint64(uint64(stats.ForwardStepCount[InternalIterCall])),
+			humanize.Count.Uint64(uint64(stats.ReverseStepCount[InternalIterCall])),
+		)
+	}
+
+	if stats.InternalStats != (InternalIteratorStats{}) {
+		s.SafeString("; ")
+		stats.InternalStats.SafeFormat(s, verb)
 	}
 	if stats.RangeKeyStats != (RangeKeyIteratorStats{}) {
-		s.SafeString(",\n(range-key-stats: ")
+		s.SafeString(", ")
-		s.Printf("(count %d), (contained points: (count %d, skipped %d)))",
+		stats.RangeKeyStats.SafeFormat(s, verb)
-			stats.RangeKeyStats.Count,
+	}
-			stats.RangeKeyStats.ContainedPoints,
+}
-			stats.RangeKeyStats.SkippedPoints)
+
+// CanDeterministicallySingleDelete takes a valid iterator and examines internal
+// state to determine if a SingleDelete deleting Iterator.Key() would
+// deterministically delete the key. CanDeterministicallySingleDelete requires
+// the iterator to be oriented in the forward direction (eg, the last
+// positioning operation must've been a First, a Seek[Prefix]GE, or a
+// Next[Prefix][WithLimit]).
+//
+// This function does not change the external position of the iterator, and all
+// positioning methods should behave the same as if it was never called. This
+// function will only return a meaningful result the first time it's invoked at
+// an iterator position. This function invalidates the iterator Value's memory,
+// and the caller must not rely on the memory safety of the previous Iterator
+// position.
+//
+// If CanDeterministicallySingleDelete returns true AND the key at the iterator
+// position is not modified between the creation of the Iterator and the commit
+// of a batch containing a SingleDelete over the key, then the caller can be
+// assured that SingleDelete is equivalent to Delete on the local engine, but it
+// may not be true on another engine that received the same writes and with
+// logically equivalent state since this engine may have collapsed multiple SETs
+// into one.
+func CanDeterministicallySingleDelete(it *Iterator) (bool, error) {
+	// This function may only be called once per external iterator position. We
+	// can validate this by checking the last positioning operation.
+	if it.lastPositioningOp == internalNextOp {
+		return false, errors.New("pebble: CanDeterministicallySingleDelete called twice")
+	}
+	validity, kind := it.internalNext()
+	var shadowedBySingleDelete bool
+	for validity == internalNextValid {
+		switch kind {
+		case InternalKeyKindDelete, InternalKeyKindDeleteSized:
+			// A DEL or DELSIZED tombstone is okay. An internal key
+			// sequence like SINGLEDEL; SET; DEL; SET can be handled
+			// deterministically. If there are SETs further down, we
+			// don't care about them.
+			return true, nil
+		case InternalKeyKindSingleDelete:
+			// A SingleDelete is okay as long as when that SingleDelete was
+			// written, it was written deterministically (eg, with its own
+			// CanDeterministicallySingleDelete check). Validate that it was
+			// written deterministically. We'll allow one set to appear after
+			// the SingleDelete.
+			shadowedBySingleDelete = true
+			validity, kind = it.internalNext()
+			continue
+		case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
+			// If we observed a single delete, it's allowed to delete 1 key.
+			// We'll keep looping to validate that the internal keys beneath the
+			// already-written single delete are copacetic.
+			if shadowedBySingleDelete {
+				shadowedBySingleDelete = false
+				validity, kind = it.internalNext()
+				continue
+			}
+			// We encountered a shadowed SET, SETWITHDEL, MERGE. A SINGLEDEL
+			// that deleted the KV at the original iterator position could
+			// result in this key becoming visible.
+			return false, nil
+		case InternalKeyKindRangeDelete:
+			// RangeDeletes are handled by the merging iterator and should never
+			// be observed by the top-level Iterator.
+			panic(errors.AssertionFailedf("pebble: unexpected range delete"))
+		case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
+			// Range keys are interleaved at the maximal sequence number and
+			// should never be observed within a user key.
+			panic(errors.AssertionFailedf("pebble: unexpected range key"))
+		default:
+			panic(errors.AssertionFailedf("pebble: unexpected key kind: %s", errors.Safe(kind)))
+		}
+	}
+	if validity == internalNextError {
+		return false, it.Error()
+	}
+	return true, nil
+}
+
+// internalNextValidity enumerates the potential outcomes of a call to
+// internalNext.
+type internalNextValidity int8
+
+const (
+	// internalNextError is returned by internalNext when an error occurred and
+	// the caller is responsible for checking iter.Error().
+	internalNextError internalNextValidity = iota
+	// internalNextExhausted is returned by internalNext when the next internal
+	// key is an internal key with a different user key than Iterator.Key().
+	internalNextExhausted
+	// internalNextValid is returned by internalNext when the internal next
+	// found a shadowed internal key with a user key equal to Iterator.Key().
+	internalNextValid
+)
+
+// internalNext advances internal Iterator state forward to expose the
+// InternalKeyKind of the next internal key with a user key equal to Key().
+//
+// internalNext is a highly specialized operation and is unlikely to be
+// generally useful. See Iterator.Next for how to reposition the iterator to the
+// next key. internalNext requires the Iterator to be at a valid position in the
+// forward direction (the last positioning operation must've been a First, a
+// Seek[Prefix]GE, or a Next[Prefix][WithLimit] and Valid() must return true).
+//
+// internalNext, unlike all other Iterator methods, exposes internal LSM state.
+// internalNext advances the Iterator's internal iterator to the next shadowed
+// key with a user key equal to Key(). When a key is overwritten or deleted, its
+// removal from the LSM occurs lazily as a part of compactions. internalNext
+// allows the caller to see whether an obsolete internal key exists with the
+// current Key(), and what it's key kind is. Note that the existence of an
+// internal key is nondeterministic and dependent on internal LSM state. These
+// semantics are unlikely to be applicable to almost all use cases.
+//
+// If internalNext finds a key that shares the same user key as Key(), it
+// returns internalNextValid and the internal key's kind. If internalNext
+// encounters an error, it returns internalNextError and the caller is expected
+// to call Iterator.Error() to retrieve it. In all other circumstances,
+// internalNext returns internalNextExhausted, indicating that there are no more
+// additional internal keys with the user key Key().
+//
+// internalNext does not change the external position of the iterator, and a
+// Next operation should behave the same as if internalNext was never called.
+// internalNext does invalidate the iterator Value's memory, and the caller must
+// not rely on the memory safety of the previous Iterator position.
+func (i *Iterator) internalNext() (internalNextValidity, base.InternalKeyKind) {
+	i.stats.ForwardStepCount[InterfaceCall]++
+	if i.err != nil {
+		return internalNextError, base.InternalKeyKindInvalid
+	} else if i.iterValidityState != IterValid {
+		return internalNextExhausted, base.InternalKeyKindInvalid
+	}
+	i.lastPositioningOp = internalNextOp
+
+	switch i.pos {
+	case iterPosCurForward:
+		i.iterKey, i.iterValue = i.iter.Next()
+		if i.iterKey == nil {
+			// We check i.iter.Error() here and return an internalNextError enum
+			// variant so that the caller does not need to check i.iter.Error()
+			// in the common case that the next internal key has a new user key.
+			if i.err = i.iter.Error(); i.err != nil {
+				return internalNextError, base.InternalKeyKindInvalid
+			}
+			i.pos = iterPosNext
+			return internalNextExhausted, base.InternalKeyKindInvalid
+		} else if i.comparer.Equal(i.iterKey.UserKey, i.key) {
+			return internalNextValid, i.iterKey.Kind()
+		}
+		i.pos = iterPosNext
+		return internalNextExhausted, base.InternalKeyKindInvalid
+	case iterPosCurReverse, iterPosCurReversePaused, iterPosPrev:
+		i.err = errors.New("switching from reverse to forward via internalNext is prohibited")
+		i.iterValidityState = IterExhausted
+		return internalNextError, base.InternalKeyKindInvalid
+	case iterPosNext, iterPosCurForwardPaused:
+		// The previous method already moved onto the next user key. This is
+		// only possible if
+		//   - the last positioning method was a call to internalNext, and we
+		//     advanced to a new user key.
+		//   - the previous non-internalNext iterator operation encountered a
+		//     range key or merge, forcing an internal Next that found a new
+		//     user key that's not equal to i.Iterator.Key().
+		return internalNextExhausted, base.InternalKeyKindInvalid
+	default:
+		panic("unreachable")
 	}
 }

vendor/github.com/cockroachdb/pebble/level_checker.go

@@ -345,7 +345,7 @@ func iterateAndCheckTombstones(
 
 type checkConfig struct {
 	logger    Logger
-	cmp       Compare
+	comparer  *Comparer
 	readState *readState
 	newIters  tableNewIters
 	seqNum    uint64
@@ -354,6 +354,9 @@ type checkConfig struct {
 	formatKey base.FormatKey
 }
 
+// cmp is shorthand for comparer.Compare.
+func (c *checkConfig) cmp(a, b []byte) int { return c.comparer.Compare(a, b) }
+
 func checkRangeTombstones(c *checkConfig) error {
 	var level int
 	var tombstones []tombstoneWithLevel
@@ -571,7 +574,7 @@ func (d *DB) CheckLevels(stats *CheckLevelsStats) error {
 
 	checkConfig := &checkConfig{
 		logger:    d.opts.Logger,
-		cmp:       d.cmp,
+		comparer:  d.opts.Comparer,
 		readState: readState,
 		newIters:  d.newIters,
 		seqNum:    seqNum,
@@ -639,7 +642,7 @@ func checkLevelsInternal(c *checkConfig) (err error) {
 		manifestIter := current.L0SublevelFiles[sublevel].Iter()
 		iterOpts := IterOptions{logger: c.logger}
 		li := &levelIter{}
-		li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters, manifestIter,
+		li.init(context.Background(), iterOpts, c.comparer, c.newIters, manifestIter,
 			manifest.L0Sublevel(sublevel), internalIterOpts{})
 		li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
 		li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext)
@@ -653,7 +656,7 @@ func checkLevelsInternal(c *checkConfig) (err error) {
 
 		iterOpts := IterOptions{logger: c.logger}
 		li := &levelIter{}
-		li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters,
+		li.init(context.Background(), iterOpts, c.comparer, c.newIters,
 			current.Levels[level].Iter(), manifest.Level(level), internalIterOpts{})
 		li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
 		li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext)

vendor/github.com/cockroachdb/pebble/level_iter.go

@@ -37,8 +37,7 @@ type tableNewIters func(
 // for the rangedel iterator returned by tableNewIters.
 func tableNewRangeDelIter(ctx context.Context, newIters tableNewIters) keyspan.TableNewSpanIter {
 	return func(file *manifest.FileMetadata, iterOptions keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) {
-		iter, rangeDelIter, err := newIters(
+		iter, rangeDelIter, err := newIters(ctx, file, nil, internalIterOpts{})
-			ctx, file, &IterOptions{RangeKeyFilters: iterOptions.RangeKeyFilters, level: iterOptions.Level}, internalIterOpts{})
 		if iter != nil {
 			_ = iter.Close()
 		}
@@ -51,6 +50,7 @@ func tableNewRangeDelIter(ctx context.Context, newIters tableNewIters) keyspan.T
 
 type internalIterOpts struct {
 	bytesIterated      *uint64
+	bufferPool         *sstable.BufferPool
 	stats              *base.InternalIteratorStats
 	boundLimitedFilter sstable.BoundLimitedBlockPropertyFilter
 }
@@ -82,10 +82,11 @@ type levelIter struct {
 	// short-lived (since they pin sstables), (b) plumbing a context into every
 	// method is very painful, (c) they do not (yet) respect context
 	// cancellation and are only used for tracing.
-	ctx    context.Context
+	ctx      context.Context
-	logger Logger
+	logger   Logger
-	cmp    Compare
+	comparer *Comparer
-	split  Split
+	cmp      Compare
+	split    Split
 	// The lower/upper bounds for iteration as specified at creation or the most
 	// recent call to SetBounds.
 	lower []byte
@@ -241,24 +242,22 @@ var _ base.InternalIterator = (*levelIter)(nil)
 // parameter if the caller is never going to call SeekPrefixGE.
 func newLevelIter(
 	opts IterOptions,
-	cmp Compare,
+	comparer *Comparer,
-	split Split,
 	newIters tableNewIters,
 	files manifest.LevelIterator,
 	level manifest.Level,
-	bytesIterated *uint64,
+	internalOpts internalIterOpts,
 ) *levelIter {
 	l := &levelIter{}
-	l.init(context.Background(), opts, cmp, split, newIters, files, level,
+	l.init(context.Background(), opts, comparer, newIters, files, level,
-		internalIterOpts{bytesIterated: bytesIterated})
+		internalOpts)
 	return l
 }
 
 func (l *levelIter) init(
 	ctx context.Context,
 	opts IterOptions,
-	cmp Compare,
+	comparer *Comparer,
-	split Split,
 	newIters tableNewIters,
 	files manifest.LevelIterator,
 	level manifest.Level,
@@ -278,8 +277,9 @@ func (l *levelIter) init(
 	l.tableOpts.UseL6Filters = opts.UseL6Filters
 	l.tableOpts.level = l.level
 	l.tableOpts.snapshotForHideObsoletePoints = opts.snapshotForHideObsoletePoints
-	l.cmp = cmp
+	l.comparer = comparer
-	l.split = split
+	l.cmp = comparer.Compare
+	l.split = comparer.Split
 	l.iterFile = nil
 	l.newIters = newIters
 	l.files = files
@@ -797,7 +797,7 @@ func (l *levelIter) SeekPrefixGE(
 		}
 		return l.verify(l.largestBoundary, base.LazyValue{})
 	}
-	// It is possible that we are here because bloom filter matching failed.  In
+	// It is possible that we are here because bloom filter matching failed. In
 	// that case it is likely that all keys matching the prefix are wholly
 	// within the current file and cannot be in the subsequent file. In that
 	// case we don't want to go to the next file, since loading and seeking in
@@ -805,7 +805,16 @@ func (l *levelIter) SeekPrefixGE(
 	// next file will defeat the optimization for the next SeekPrefixGE that is
 	// called with flags.TrySeekUsingNext(), since for sparse key spaces it is
 	// likely that the next key will also be contained in the current file.
-	if n := l.split(l.iterFile.LargestPointKey.UserKey); l.cmp(prefix, l.iterFile.LargestPointKey.UserKey[:n]) < 0 {
+	var n int
+	if l.split != nil {
+		// If the split function is specified, calculate the prefix length accordingly.
+		n = l.split(l.iterFile.LargestPointKey.UserKey)
+	} else {
+		// If the split function is not specified, the entire key is used as the
+		// prefix. This case can occur when getIter uses SeekPrefixGE.
+		n = len(l.iterFile.LargestPointKey.UserKey)
+	}
+	if l.cmp(prefix, l.iterFile.LargestPointKey.UserKey[:n]) < 0 {
 		return nil, base.LazyValue{}
 	}
 	return l.verify(l.skipEmptyFileForward())
@@ -1075,7 +1084,9 @@ func (l *levelIter) skipEmptyFileForward() (*InternalKey, base.LazyValue) {
 			if *l.rangeDelIterPtr != nil && l.filteredIter != nil &&
 				l.filteredIter.MaybeFilteredKeys() {
 				l.largestBoundary = &l.iterFile.Largest
-				l.boundaryContext.isIgnorableBoundaryKey = true
+				if l.boundaryContext != nil {
+					l.boundaryContext.isIgnorableBoundaryKey = true
+				}
 				return l.largestBoundary, base.LazyValue{}
 			}
 		}
@@ -1164,7 +1175,9 @@ func (l *levelIter) skipEmptyFileBackward() (*InternalKey, base.LazyValue) {
 			// the next file.
 			if *l.rangeDelIterPtr != nil && l.filteredIter != nil && l.filteredIter.MaybeFilteredKeys() {
 				l.smallestBoundary = &l.iterFile.Smallest
-				l.boundaryContext.isIgnorableBoundaryKey = true
+				if l.boundaryContext != nil {
+					l.boundaryContext.isIgnorableBoundaryKey = true
+				}
 				return l.smallestBoundary, base.LazyValue{}
 			}
 		}

vendor/github.com/cockroachdb/pebble/mem_table.go

@@ -10,12 +10,12 @@ import (
 	"os"
 	"sync"
 	"sync/atomic"
-	"unsafe"
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/arenaskl"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/keyspan"
+	"github.com/cockroachdb/pebble/internal/manual"
 	"github.com/cockroachdb/pebble/internal/rangedel"
 	"github.com/cockroachdb/pebble/internal/rangekey"
 )
@@ -85,7 +85,16 @@ type memTable struct {
 	rangeKeys  keySpanCache
 	// The current logSeqNum at the time the memtable was created. This is
 	// guaranteed to be less than or equal to any seqnum stored in the memtable.
-	logSeqNum uint64
+	logSeqNum                    uint64
+	releaseAccountingReservation func()
+}
+
+func (m *memTable) free() {
+	if m != nil {
+		m.releaseAccountingReservation()
+		manual.Free(m.arenaBuf)
+		m.arenaBuf = nil
+	}
 }
 
 // memTableOptions holds configuration used when creating a memTable. All of
@@ -93,9 +102,10 @@ type memTable struct {
 // which is used by tests.
 type memTableOptions struct {
 	*Options
-	arenaBuf  []byte
+	arenaBuf                     []byte
-	size      int
+	size                         int
-	logSeqNum uint64
+	logSeqNum                    uint64
+	releaseAccountingReservation func()
 }
 
 func checkMemTable(obj interface{}) {
@@ -110,16 +120,22 @@ func checkMemTable(obj interface{}) {
 // Options.MemTableSize is used instead.
 func newMemTable(opts memTableOptions) *memTable {
 	opts.Options = opts.Options.EnsureDefaults()
-	if opts.size == 0 {
+	m := new(memTable)
-		opts.size = opts.MemTableSize
+	m.init(opts)
-	}
+	return m
+}
 
-	m := &memTable{
+func (m *memTable) init(opts memTableOptions) {
-		cmp:       opts.Comparer.Compare,
+	if opts.size == 0 {
-		formatKey: opts.Comparer.FormatKey,
+		opts.size = int(opts.MemTableSize)
-		equal:     opts.Comparer.Equal,
+	}
-		arenaBuf:  opts.arenaBuf,
+	*m = memTable{
-		logSeqNum: opts.logSeqNum,
+		cmp:                          opts.Comparer.Compare,
+		formatKey:                    opts.Comparer.FormatKey,
+		equal:                        opts.Comparer.Equal,
+		arenaBuf:                     opts.arenaBuf,
+		logSeqNum:                    opts.logSeqNum,
+		releaseAccountingReservation: opts.releaseAccountingReservation,
 	}
 	m.writerRefs.Store(1)
 	m.tombstones = keySpanCache{
@@ -144,7 +160,6 @@ func newMemTable(opts memTableOptions) *memTable {
 	m.rangeDelSkl.Reset(arena, m.cmp)
 	m.rangeKeySkl.Reset(arena, m.cmp)
 	m.reserved = arena.Size()
-	return m
 }
 
 func (m *memTable) writerRef() {
@@ -154,7 +169,8 @@ func (m *memTable) writerRef() {
 	}
 }
 
-func (m *memTable) writerUnref() bool {
+// writerUnref drops a ref on the memtable. Returns true if this was the last ref.
+func (m *memTable) writerUnref() (wasLastRef bool) {
 	switch v := m.writerRefs.Add(-1); {
 	case v < 0:
 		panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v))
@@ -165,6 +181,7 @@ func (m *memTable) writerUnref() bool {
 	}
 }
 
+// readyForFlush is part of the flushable interface.
 func (m *memTable) readyForFlush() bool {
 	return m.writerRefs.Load() == 0
 }
@@ -194,11 +211,13 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error {
 	var tombstoneCount, rangeKeyCount uint32
 	startSeqNum := seqNum
 	for r := batch.Reader(); ; seqNum++ {
-		kind, ukey, value, ok := r.Next()
+		kind, ukey, value, ok, err := r.Next()
 		if !ok {
+			if err != nil {
+				return err
+			}
 			break
 		}
-		var err error
 		ikey := base.MakeInternalKey(ukey, seqNum, kind)
 		switch kind {
 		case InternalKeyKindRangeDelete:
@@ -233,17 +252,19 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error {
 	return nil
 }
 
-// newIter returns an iterator that is unpositioned (Iterator.Valid() will
+// newIter is part of the flushable interface. It returns an iterator that is
-// return false). The iterator can be positioned via a call to SeekGE,
+// unpositioned (Iterator.Valid() will return false). The iterator can be
-// SeekLT, First or Last.
+// positioned via a call to SeekGE, SeekLT, First or Last.
 func (m *memTable) newIter(o *IterOptions) internalIterator {
 	return m.skl.NewIter(o.GetLowerBound(), o.GetUpperBound())
 }
 
+// newFlushIter is part of the flushable interface.
 func (m *memTable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator {
 	return m.skl.NewFlushIter(bytesFlushed)
 }
 
+// newRangeDelIter is part of the flushable interface.
 func (m *memTable) newRangeDelIter(*IterOptions) keyspan.FragmentIterator {
 	tombstones := m.tombstones.get()
 	if tombstones == nil {
@@ -252,6 +273,7 @@ func (m *memTable) newRangeDelIter(*IterOptions) keyspan.FragmentIterator {
 	return keyspan.NewIter(m.cmp, tombstones)
 }
 
+// newRangeKeyIter is part of the flushable interface.
 func (m *memTable) newRangeKeyIter(*IterOptions) keyspan.FragmentIterator {
 	rangeKeys := m.rangeKeys.get()
 	if rangeKeys == nil {
@@ -260,8 +282,9 @@ func (m *memTable) newRangeKeyIter(*IterOptions) keyspan.FragmentIterator {
 	return keyspan.NewIter(m.cmp, rangeKeys)
 }
 
+// containsRangeKeys is part of the flushable interface.
 func (m *memTable) containsRangeKeys() bool {
-	return atomic.LoadUint32(&m.rangeKeys.atomicCount) > 0
+	return m.rangeKeys.count.Load() > 0
 }
 
 func (m *memTable) availBytes() uint32 {
@@ -275,10 +298,12 @@ func (m *memTable) availBytes() uint32 {
 	return a.Capacity() - m.reserved
 }
 
+// inuseBytes is part of the flushable interface.
 func (m *memTable) inuseBytes() uint64 {
 	return uint64(m.skl.Size() - memTableEmptySize)
 }
 
+// totalBytes is part of the flushable interface.
 func (m *memTable) totalBytes() uint64 {
 	return uint64(m.skl.Arena().Capacity())
 }
@@ -355,8 +380,8 @@ func (f *keySpanFrags) get(
 // invalidated whenever a key of the same kind is added to a memTable, and
 // populated when empty when a span iterator of that key kind is created.
 type keySpanCache struct {
-	atomicCount   uint32
+	count         atomic.Uint32
-	frags         unsafe.Pointer
+	frags         atomic.Pointer[keySpanFrags]
 	cmp           Compare
 	formatKey     base.FormatKey
 	constructSpan constructSpan
@@ -366,23 +391,20 @@ type keySpanCache struct {
 // Invalidate the current set of cached spans, indicating the number of
 // spans that were added.
 func (c *keySpanCache) invalidate(count uint32) {
-	newCount := atomic.AddUint32(&c.atomicCount, count)
+	newCount := c.count.Add(count)
 	var frags *keySpanFrags
 
 	for {
-		oldPtr := atomic.LoadPointer(&c.frags)
+		oldFrags := c.frags.Load()
-		if oldPtr != nil {
+		if oldFrags != nil && oldFrags.count >= newCount {
-			oldFrags := (*keySpanFrags)(oldPtr)
+			// Someone else invalidated the cache before us and their invalidation
-			if oldFrags.count >= newCount {
+			// subsumes ours.
-				// Someone else invalidated the cache before us and their invalidation
+			break
-				// subsumes ours.
-				break
-			}
 		}
 		if frags == nil {
 			frags = &keySpanFrags{count: newCount}
 		}
-		if atomic.CompareAndSwapPointer(&c.frags, oldPtr, unsafe.Pointer(frags)) {
+		if c.frags.CompareAndSwap(oldFrags, frags) {
 			// We successfully invalidated the cache.
 			break
 		}
@@ -391,7 +413,7 @@ func (c *keySpanCache) invalidate(count uint32) {
 }
 
 func (c *keySpanCache) get() []keyspan.Span {
-	frags := (*keySpanFrags)(atomic.LoadPointer(&c.frags))
+	frags := c.frags.Load()
 	if frags == nil {
 		return nil
 	}

vendor/github.com/cockroachdb/pebble/merging_iter.go

@@ -27,6 +27,10 @@ type mergingIterLevel struct {
 	// iterKey and iterValue cache the current key and value iter are pointed at.
 	iterKey   *InternalKey
 	iterValue base.LazyValue
+	// levelIter is non-nil if this level's iter is ultimately backed by a
+	// *levelIter. The handle in iter may have wrapped the levelIter with
+	// intermediary internalIterator implementations.
+	levelIter *levelIter
 
 	// levelIterBoundaryContext's fields are set when using levelIter, in order
 	// to surface sstable boundary keys and file-level context. See levelIter
@@ -1371,9 +1375,9 @@ func (m *mergingIter) DebugString() string {
 }
 
 func (m *mergingIter) ForEachLevelIter(fn func(li *levelIter) bool) {
-	for _, iter := range m.levels {
+	for _, ml := range m.levels {
-		if li, ok := iter.iter.(*levelIter); ok {
+		if ml.levelIter != nil {
-			if done := fn(li); done {
+			if done := fn(ml.levelIter); done {
 				break
 			}
 		}

vendor/github.com/cockroachdb/pebble/metrics.go

@@ -6,12 +6,13 @@ package pebble
 
 import (
 	"fmt"
-	"strings"
+	"math"
 	"time"
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/cockroachdb/pebble/internal/humanize"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
 	"github.com/cockroachdb/pebble/record"
 	"github.com/cockroachdb/pebble/sstable"
 	"github.com/cockroachdb/redact"
@@ -28,13 +29,10 @@ type FilterMetrics = sstable.FilterMetrics
 // comment in base.
 type ThroughputMetric = base.ThroughputMetric
 
-func formatCacheMetrics(w redact.SafePrinter, m *CacheMetrics, name redact.SafeString) {
+// SecondaryCacheMetrics holds metrics for the persistent secondary cache
-	w.Printf("%7s %9s %7s %6.1f%%  (score == hit-rate)\n",
+// that caches commonly accessed blocks from blob storage on a local
-		name,
+// file system.
-		humanize.SI.Int64(m.Count),
+type SecondaryCacheMetrics = sharedcache.Metrics
-		humanize.IEC.Int64(m.Size),
-		redact.Safe(hitRate(m.Hits, m.Misses)))
-}
 
 // LevelMetrics holds per-level metrics such as the number of files and total
 // size of the files, and compaction related metrics.
@@ -46,9 +44,14 @@ type LevelMetrics struct {
 	Sublevels int32
 	// The total number of files in the level.
 	NumFiles int64
+	// The total number of virtual sstables in the level.
+	NumVirtualFiles uint64
 	// The total size in bytes of the files in the level.
 	Size int64
-	// The level's compaction score.
+	// The total size of the virtual sstables in the level.
+	VirtualSize uint64
+	// The level's compaction score. This is the compensatedScoreRatio in the
+	// candidateLevelInfo.
 	Score float64
 	// The number of incoming bytes from other levels read during
 	// compactions. This excludes bytes moved and bytes ingested. For L0 this is
@@ -79,6 +82,18 @@ type LevelMetrics struct {
 	TablesIngested uint64
 	// The number of sstables moved to this level by a "move" compaction.
 	TablesMoved uint64
+
+	MultiLevel struct {
+		// BytesInTop are the total bytes in a multilevel compaction coming from the top level.
+		BytesInTop uint64
+
+		// BytesIn, exclusively for multiLevel compactions.
+		BytesIn uint64
+
+		// BytesRead, exclusively for multilevel compactions.
+		BytesRead uint64
+	}
+
 	// Additional contains misc additional metrics that are not always printed.
 	Additional struct {
 		// The sum of Properties.ValueBlocksSize for all the sstables in this
@@ -96,6 +111,8 @@ type LevelMetrics struct {
 // Add updates the counter metrics for the level.
 func (m *LevelMetrics) Add(u *LevelMetrics) {
 	m.NumFiles += u.NumFiles
+	m.NumVirtualFiles += u.NumVirtualFiles
+	m.VirtualSize += u.VirtualSize
 	m.Size += u.Size
 	m.BytesIn += u.BytesIn
 	m.BytesIngested += u.BytesIngested
@@ -107,6 +124,9 @@ func (m *LevelMetrics) Add(u *LevelMetrics) {
 	m.TablesFlushed += u.TablesFlushed
 	m.TablesIngested += u.TablesIngested
 	m.TablesMoved += u.TablesMoved
+	m.MultiLevel.BytesInTop += u.MultiLevel.BytesInTop
+	m.MultiLevel.BytesRead += u.MultiLevel.BytesRead
+	m.MultiLevel.BytesIn += u.MultiLevel.BytesIn
 	m.Additional.BytesWrittenDataBlocks += u.Additional.BytesWrittenDataBlocks
 	m.Additional.BytesWrittenValueBlocks += u.Additional.BytesWrittenValueBlocks
 	m.Additional.ValueBlocksSize += u.Additional.ValueBlocksSize
@@ -121,32 +141,6 @@ func (m *LevelMetrics) WriteAmp() float64 {
 	return float64(m.BytesFlushed+m.BytesCompacted) / float64(m.BytesIn)
 }
 
-// format generates a string of the receiver's metrics, formatting it into the
-// supplied buffer.
-func (m *LevelMetrics) format(
-	w redact.SafePrinter, score redact.SafeValue, includeValueBlocksSize bool,
-) {
-	w.Printf("%9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7d %7.1f",
-		redact.Safe(m.NumFiles),
-		humanize.IEC.Int64(m.Size),
-		score,
-		humanize.IEC.Uint64(m.BytesIn),
-		humanize.IEC.Uint64(m.BytesIngested),
-		humanize.SI.Uint64(m.TablesIngested),
-		humanize.IEC.Uint64(m.BytesMoved),
-		humanize.SI.Uint64(m.TablesMoved),
-		humanize.IEC.Uint64(m.BytesFlushed+m.BytesCompacted),
-		humanize.SI.Uint64(m.TablesFlushed+m.TablesCompacted),
-		humanize.IEC.Uint64(m.BytesRead),
-		redact.Safe(m.Sublevels),
-		redact.Safe(m.WriteAmp()))
-	if includeValueBlocksSize {
-		w.Printf(" %7s\n", humanize.IEC.Uint64(m.Additional.ValueBlocksSize))
-	} else {
-		w.SafeString("\n")
-	}
-}
-
 // Metrics holds metrics for various subsystems of the DB such as the Cache,
 // Compactions, WAL, and per-Level metrics.
 //
@@ -158,14 +152,15 @@ type Metrics struct {
 
 	Compact struct {
 		// The total number of compactions, and per-compaction type counts.
-		Count            int64
+		Count             int64
-		DefaultCount     int64
+		DefaultCount      int64
-		DeleteOnlyCount  int64
+		DeleteOnlyCount   int64
-		ElisionOnlyCount int64
+		ElisionOnlyCount  int64
-		MoveCount        int64
+		MoveCount         int64
-		ReadCount        int64
+		ReadCount         int64
-		RewriteCount     int64
+		RewriteCount      int64
-		MultiLevelCount  int64
+		MultiLevelCount   int64
+		CounterLevelCount int64
 		// An estimate of the number of bytes that need to be compacted for the LSM
 		// to reach a stable state.
 		EstimatedDebt uint64
@@ -218,7 +213,10 @@ type Metrics struct {
 		// The count of memtables.
 		Count int64
 		// The number of bytes present in zombie memtables which are no longer
-		// referenced by the current DB state but are still in use by an iterator.
+		// referenced by the current DB state. An unbounded number of memtables
+		// may be zombie if they're still in use by an iterator. One additional
+		// memtable may be zombie if it's no longer in use and waiting to be
+		// recycled.
 		ZombieSize uint64
 		// The count of zombie memtables.
 		ZombieCount int64
@@ -261,6 +259,10 @@ type Metrics struct {
 		ZombieSize uint64
 		// The count of zombie tables.
 		ZombieCount int64
+		// The count of the backing sstables.
+		BackingTableCount uint64
+		// The sum of the sizes of the all of the backing sstables.
+		BackingTableSize uint64
 	}
 
 	TableCache CacheMetrics
@@ -294,6 +296,8 @@ type Metrics struct {
 		record.LogWriterMetrics
 	}
 
+	SecondaryCacheMetrics SecondaryCacheMetrics
+
 	private struct {
 		optionsFileSize  uint64
 		manifestFileSize uint64
@@ -307,6 +311,13 @@ var (
 		prometheus.LinearBuckets(0.0, float64(time.Microsecond*100), 50),
 		prometheus.ExponentialBucketsRange(float64(time.Millisecond*5), float64(10*time.Second), 50)...,
 	)
+
+	// SecondaryCacheIOBuckets exported to enable exporting from package pebble to
+	// enable exporting metrics with below buckets in CRDB.
+	SecondaryCacheIOBuckets = sharedcache.IOBuckets
+	// SecondaryCacheChannelWriteBuckets exported to enable exporting from package
+	// pebble to enable exporting metrics with below buckets in CRDB.
+	SecondaryCacheChannelWriteBuckets = sharedcache.ChannelWriteBuckets
 )
 
 // DiskSpaceUsage returns the total disk space used by the database in bytes,
@@ -326,12 +337,25 @@ func (m *Metrics) DiskSpaceUsage() uint64 {
 	return usageBytes
 }
 
-func (m *Metrics) levelSizes() [numLevels]int64 {
+// NumVirtual is the number of virtual sstables in the latest version
-	var sizes [numLevels]int64
+// summed over every level in the lsm.
-	for i := 0; i < len(sizes); i++ {
+func (m *Metrics) NumVirtual() uint64 {
-		sizes[i] = m.Levels[i].Size
+	var n uint64
+	for _, level := range m.Levels {
+		n += level.NumVirtualFiles
 	}
-	return sizes
+	return n
+}
+
+// VirtualSize is the sum of the sizes of the virtual sstables in the
+// latest version. BackingTableSize - VirtualSize gives an estimate for
+// the space amplification caused by not compacting virtual sstables.
+func (m *Metrics) VirtualSize() uint64 {
+	var size uint64
+	for _, level := range m.Levels {
+		size += level.VirtualSize
+	}
+	return size
 }
 
 // ReadAmp returns the current read amplification of the database.
@@ -362,59 +386,34 @@ func (m *Metrics) Total() LevelMetrics {
 	return total
 }
 
-const notApplicable = redact.SafeString("-")
+// String pretty-prints the metrics as below:
-
-func (m *Metrics) formatWAL(w redact.SafePrinter) {
-	var writeAmp float64
-	if m.WAL.BytesIn > 0 {
-		writeAmp = float64(m.WAL.BytesWritten) / float64(m.WAL.BytesIn)
-	}
-	w.Printf("    WAL %9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7.1f\n",
-		redact.Safe(m.WAL.Files),
-		humanize.Uint64(m.WAL.Size),
-		notApplicable,
-		humanize.Uint64(m.WAL.BytesIn),
-		notApplicable,
-		notApplicable,
-		notApplicable,
-		notApplicable,
-		humanize.Uint64(m.WAL.BytesWritten),
-		notApplicable,
-		notApplicable,
-		notApplicable,
-		redact.Safe(writeAmp))
-}
-
-// String pretty-prints the metrics, showing a line for the WAL, a line per-level, and
-// a total:
 //
-//	  __level_____count____size___score______in__ingest(sz_cnt)____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp
+//	      |                             |       |       |   ingested   |     moved    |    written   |       |    amp
-//	    WAL         1    28 B       -    17 B       -       -       -       -    56 B       -       -       -     3.3
+//	level | tables  size val-bl vtables | score |   in  | tables  size | tables  size | tables  size |  read |   r   w
-//	      0         1   770 B    0.25    28 B     0 B       0     0 B       0   770 B       1     0 B       1    27.5
+//	------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+---------
-//	      1         0     0 B    0.00     0 B     0 B       0     0 B       0     0 B       0     0 B       0     0.0
+//	    0 |   101   102B     0B       0 | 103.0 |  104B |   112   104B |   113   106B |   221   217B |  107B |   1  2.1
-//	      2         0     0 B    0.00     0 B     0 B       0     0 B       0     0 B       0     0 B       0     0.0
+//	    1 |   201   202B     0B       0 | 203.0 |  204B |   212   204B |   213   206B |   421   417B |  207B |   2  2.0
-//	      3         0     0 B    0.00     0 B     0 B       0     0 B       0     0 B       0     0 B       0     0.0
+//	    2 |   301   302B     0B       0 | 303.0 |  304B |   312   304B |   313   306B |   621   617B |  307B |   3  2.0
-//	      4         0     0 B    0.00     0 B     0 B       0     0 B       0     0 B       0     0 B       0     0.0
+//	    3 |   401   402B     0B       0 | 403.0 |  404B |   412   404B |   413   406B |   821   817B |  407B |   4  2.0
-//	      5         0     0 B    0.00     0 B     0 B       0     0 B       0     0 B       0     0 B       0     0.0
+//	    4 |   501   502B     0B       0 | 503.0 |  504B |   512   504B |   513   506B |  1.0K  1017B |  507B |   5  2.0
-//	      6         0     0 B       -     0 B     0 B       0     0 B       0     0 B       0     0 B       0     0.0
+//	    5 |   601   602B     0B       0 | 603.0 |  604B |   612   604B |   613   606B |  1.2K  1.2KB |  607B |   6  2.0
-//	  total         1   770 B       -    56 B     0 B       0     0 B       0   826 B       1     0 B       1    14.8
+//	    6 |   701   702B     0B       0 |     - |  704B |   712   704B |   713   706B |  1.4K  1.4KB |  707B |   7  2.0
-//	  flush         1                             0 B       0       0  (ingest = ingested-as-flushable, move = tables-ingested)
+//	total |  2.8K  2.7KB     0B       0 |     - | 2.8KB |  2.9K  2.8KB |  2.9K  2.8KB |  5.7K  8.4KB | 2.8KB |  28  3.0
-//	compact         0     0 B     0 B       0                          (size == estimated-debt, score = in-progress-bytes, in = num-in-progress)
+//	-------------------------------------------------------------------------------------------------------------------
-//	  ctype         0       0       0       0       0       0       0  (default, delete, elision, move, read, rewrite, multi-level)
+//	WAL: 22 files (24B)  in: 25B  written: 26B (4% overhead)
-//	 memtbl         1   256 K
+//	Flushes: 8
-//	zmemtbl         1   256 K
+//	Compactions: 5  estimated debt: 6B  in progress: 2 (7B)
-//	   ztbl         0     0 B
+//	default: 27  delete: 28  elision: 29  move: 30  read: 31  rewrite: 32  multi-level: 33
-//	 bcache         4   697 B    0.0%  (score == hit-rate)
+//	MemTables: 12 (11B)  zombie: 14 (13B)
-//	 tcache         1   696 B    0.0%  (score == hit-rate)
+//	Zombie tables: 16 (15B)
-//	  snaps         0       -       0  (score == earliest seq num)
+//	Backing tables: 0 (0B)
-//	 titers         1
+//	Block cache: 2 entries (1B)  hit rate: 42.9%
-//	 filter         -       -    0.0%  (score == utility)
+//	Table cache: 18 entries (17B)  hit rate: 48.7%
-//
+//	Secondary cache: 40 entries (40B)  hit rate: 49.9%
-// The WAL "in" metric is the size of the batches written to the WAL. The WAL
+//	Snapshots: 4  earliest seq num: 1024
-// "write" metric is the size of the physical data written to the WAL which
+//	Table iters: 21
-// includes record fragment overhead. Write amplification is computed as
+//	Filter utility: 47.4%
-// bytes-written / bytes-in, except for the total row where bytes-in is
+//	Ingestions: 27  as flushable: 36 (34B in 35 tables)
-// replaced with WAL-bytes-written + bytes-ingested.
 func (m *Metrics) String() string {
 	return redact.StringWithoutMarkers(m)
 }
@@ -435,29 +434,84 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 	// width specifiers. When the issue is fixed, we can convert these to
 	// RedactableStrings. https://github.com/cockroachdb/redact/issues/17
 
-	haveValueBlocks := false
+	multiExists := m.Compact.MultiLevelCount > 0
-	var valueBlocksHeading redact.SafeString
+	appendIfMulti := func(line redact.SafeString) {
-	for level := 0; level < numLevels; level++ {
+		if multiExists {
-		if m.Levels[level].Additional.ValueBlocksSize > 0 {
+			w.SafeString(line)
-			haveValueBlocks = true
-			valueBlocksHeading = "__val-bl"
-			break
 		}
 	}
+	newline := func() {
+		w.SafeString("\n")
+	}
+
+	w.SafeString("      |                             |       |       |   ingested   |     moved    |    written   |       |    amp")
+	appendIfMulti("   |     multilevel")
+	newline()
+	w.SafeString("level | tables  size val-bl vtables | score |   in  | tables  size | tables  size | tables  size |  read |   r   w")
+	appendIfMulti("  |    top   in  read")
+	newline()
+	w.SafeString("------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+---------")
+	appendIfMulti("-+------------------")
+	newline()
+
+	// formatRow prints out a row of the table.
+	formatRow := func(m *LevelMetrics, score float64) {
+		scoreStr := "-"
+		if !math.IsNaN(score) {
+			// Try to keep the string no longer than 5 characters.
+			switch {
+			case score < 99.995:
+				scoreStr = fmt.Sprintf("%.2f", score)
+			case score < 999.95:
+				scoreStr = fmt.Sprintf("%.1f", score)
+			default:
+				scoreStr = fmt.Sprintf("%.0f", score)
+			}
+		}
+		var wampStr string
+		if wamp := m.WriteAmp(); wamp > 99.5 {
+			wampStr = fmt.Sprintf("%.0f", wamp)
+		} else {
+			wampStr = fmt.Sprintf("%.1f", wamp)
+		}
+
+		w.Printf("| %5s %6s %6s %7s | %5s | %5s | %5s %6s | %5s %6s | %5s %6s | %5s | %3d %4s",
+			humanize.Count.Int64(m.NumFiles),
+			humanize.Bytes.Int64(m.Size),
+			humanize.Bytes.Uint64(m.Additional.ValueBlocksSize),
+			humanize.Count.Uint64(m.NumVirtualFiles),
+			redact.Safe(scoreStr),
+			humanize.Bytes.Uint64(m.BytesIn),
+			humanize.Count.Uint64(m.TablesIngested),
+			humanize.Bytes.Uint64(m.BytesIngested),
+			humanize.Count.Uint64(m.TablesMoved),
+			humanize.Bytes.Uint64(m.BytesMoved),
+			humanize.Count.Uint64(m.TablesFlushed+m.TablesCompacted),
+			humanize.Bytes.Uint64(m.BytesFlushed+m.BytesCompacted),
+			humanize.Bytes.Uint64(m.BytesRead),
+			redact.Safe(m.Sublevels),
+			redact.Safe(wampStr))
+
+		if multiExists {
+			w.Printf(" | %5s %5s %5s",
+				humanize.Bytes.Uint64(m.MultiLevel.BytesInTop),
+				humanize.Bytes.Uint64(m.MultiLevel.BytesIn),
+				humanize.Bytes.Uint64(m.MultiLevel.BytesRead))
+		}
+		newline()
+	}
+
 	var total LevelMetrics
-	w.Printf("__level_____count____size___score______in__ingest(sz_cnt)"+
-		"____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp%s\n", valueBlocksHeading)
-	m.formatWAL(w)
 	for level := 0; level < numLevels; level++ {
 		l := &m.Levels[level]
-		w.Printf("%7d ", redact.Safe(level))
+		w.Printf("%5d ", redact.Safe(level))
 
 		// Format the score.
-		var score redact.SafeValue = notApplicable
+		score := math.NaN()
 		if level < numLevels-1 {
-			score = redact.Safe(fmt.Sprintf("%0.2f", l.Score))
+			score = l.Score
 		}
-		l.format(w, score, haveValueBlocks)
+		formatRow(l, score)
 		total.Add(l)
 		total.Sublevels += l.Sublevels
 	}
@@ -467,60 +521,105 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 	// the bytes written to the log and bytes written externally and then
 	// ingested.
 	total.BytesFlushed += total.BytesIn
-	w.SafeString("  total ")
+	w.SafeString("total ")
-	total.format(w, notApplicable, haveValueBlocks)
+	formatRow(&total, math.NaN())
 
-	w.Printf("  flush %9d %31s %7d %7d  %s\n",
+	w.SafeString("-------------------------------------------------------------------------------------------------------------------")
-		redact.Safe(m.Flush.Count),
+	appendIfMulti("--------------------")
-		humanize.IEC.Uint64(m.Flush.AsIngestBytes),
+	newline()
-		redact.Safe(m.Flush.AsIngestTableCount),
+	w.Printf("WAL: %d files (%s)  in: %s  written: %s (%.0f%% overhead)\n",
-		redact.Safe(m.Flush.AsIngestCount),
+		redact.Safe(m.WAL.Files),
-		redact.SafeString(`(ingest = tables-ingested, move = ingested-as-flushable)`))
+		humanize.Bytes.Uint64(m.WAL.Size),
-	w.Printf("compact %9d %7s %7s %7d %s %s\n",
+		humanize.Bytes.Uint64(m.WAL.BytesIn),
+		humanize.Bytes.Uint64(m.WAL.BytesWritten),
+		redact.Safe(percent(int64(m.WAL.BytesWritten)-int64(m.WAL.BytesIn), int64(m.WAL.BytesIn))))
+
+	w.Printf("Flushes: %d\n", redact.Safe(m.Flush.Count))
+
+	w.Printf("Compactions: %d  estimated debt: %s  in progress: %d (%s)\n",
 		redact.Safe(m.Compact.Count),
-		humanize.IEC.Uint64(m.Compact.EstimatedDebt),
+		humanize.Bytes.Uint64(m.Compact.EstimatedDebt),
-		humanize.IEC.Int64(m.Compact.InProgressBytes),
 		redact.Safe(m.Compact.NumInProgress),
-		redact.SafeString(strings.Repeat(" ", 24)),
+		humanize.Bytes.Int64(m.Compact.InProgressBytes))
-		redact.SafeString(`(size == estimated-debt, score = in-progress-bytes, in = num-in-progress)`))
+
-	w.Printf("  ctype %9d %7d %7d %7d %7d %7d %7d  %s\n",
+	w.Printf("             default: %d  delete: %d  elision: %d  move: %d  read: %d  rewrite: %d  multi-level: %d\n",
 		redact.Safe(m.Compact.DefaultCount),
 		redact.Safe(m.Compact.DeleteOnlyCount),
 		redact.Safe(m.Compact.ElisionOnlyCount),
 		redact.Safe(m.Compact.MoveCount),
 		redact.Safe(m.Compact.ReadCount),
 		redact.Safe(m.Compact.RewriteCount),
-		redact.Safe(m.Compact.MultiLevelCount),
+		redact.Safe(m.Compact.MultiLevelCount))
-		redact.SafeString(`(default, delete, elision, move, read, rewrite, multi-level)`))
+
-	w.Printf(" memtbl %9d %7s\n",
+	w.Printf("MemTables: %d (%s)  zombie: %d (%s)\n",
 		redact.Safe(m.MemTable.Count),
-		humanize.IEC.Uint64(m.MemTable.Size))
+		humanize.Bytes.Uint64(m.MemTable.Size),
-	w.Printf("zmemtbl %9d %7s\n",
 		redact.Safe(m.MemTable.ZombieCount),
-		humanize.IEC.Uint64(m.MemTable.ZombieSize))
+		humanize.Bytes.Uint64(m.MemTable.ZombieSize))
-	w.Printf("   ztbl %9d %7s\n",
+
+	w.Printf("Zombie tables: %d (%s)\n",
 		redact.Safe(m.Table.ZombieCount),
-		humanize.IEC.Uint64(m.Table.ZombieSize))
+		humanize.Bytes.Uint64(m.Table.ZombieSize))
-	formatCacheMetrics(w, &m.BlockCache, "bcache")
+
-	formatCacheMetrics(w, &m.TableCache, "tcache")
+	w.Printf("Backing tables: %d (%s)\n",
-	w.Printf("  snaps %9d %7s %7d  (score == earliest seq num)\n",
+		redact.Safe(m.Table.BackingTableCount),
+		humanize.Bytes.Uint64(m.Table.BackingTableSize))
+	w.Printf("Virtual tables: %d (%s)\n",
+		redact.Safe(m.NumVirtual()),
+		humanize.Bytes.Uint64(m.VirtualSize()))
+
+	formatCacheMetrics := func(m *CacheMetrics, name redact.SafeString) {
+		w.Printf("%s: %s entries (%s)  hit rate: %.1f%%\n",
+			name,
+			humanize.Count.Int64(m.Count),
+			humanize.Bytes.Int64(m.Size),
+			redact.Safe(hitRate(m.Hits, m.Misses)))
+	}
+	formatCacheMetrics(&m.BlockCache, "Block cache")
+	formatCacheMetrics(&m.TableCache, "Table cache")
+
+	formatSharedCacheMetrics := func(w redact.SafePrinter, m *SecondaryCacheMetrics, name redact.SafeString) {
+		w.Printf("%s: %s entries (%s)  hit rate: %.1f%%\n",
+			name,
+			humanize.Count.Int64(m.Count),
+			humanize.Bytes.Int64(m.Size),
+			redact.Safe(hitRate(m.ReadsWithFullHit, m.ReadsWithPartialHit+m.ReadsWithNoHit)))
+	}
+	formatSharedCacheMetrics(w, &m.SecondaryCacheMetrics, "Secondary cache")
+
+	w.Printf("Snapshots: %d  earliest seq num: %d\n",
 		redact.Safe(m.Snapshots.Count),
-		notApplicable,
 		redact.Safe(m.Snapshots.EarliestSeqNum))
-	w.Printf(" titers %9d\n", redact.Safe(m.TableIters))
+
-	w.Printf(" filter %9s %7s %6.1f%%  (score == utility)\n",
+	w.Printf("Table iters: %d\n", redact.Safe(m.TableIters))
-		notApplicable,
+	w.Printf("Filter utility: %.1f%%\n", redact.Safe(hitRate(m.Filter.Hits, m.Filter.Misses)))
-		notApplicable,
+	w.Printf("Ingestions: %d  as flushable: %d (%s in %d tables)\n",
-		redact.Safe(hitRate(m.Filter.Hits, m.Filter.Misses)))
-	w.Printf(" ingest %9d\n",
 		redact.Safe(m.Ingest.Count),
-	)
+		redact.Safe(m.Flush.AsIngestCount),
+		humanize.Bytes.Uint64(m.Flush.AsIngestBytes),
+		redact.Safe(m.Flush.AsIngestTableCount))
 }
 
 func hitRate(hits, misses int64) float64 {
-	sum := hits + misses
+	return percent(hits, hits+misses)
-	if sum == 0 {
+}
+
+func percent(numerator, denominator int64) float64 {
+	if denominator == 0 {
 		return 0
 	}
-	return 100 * float64(hits) / float64(sum)
+	return 100 * float64(numerator) / float64(denominator)
+}
+
+// StringForTests is identical to m.String() on 64-bit platforms. It is used to
+// provide a platform-independent result for tests.
+func (m *Metrics) StringForTests() string {
+	mCopy := *m
+	if math.MaxInt == math.MaxInt32 {
+		// This is the difference in Sizeof(sstable.Reader{})) between 64 and 32 bit
+		// platforms.
+		const tableCacheSizeAdjustment = 212
+		mCopy.TableCache.Size += mCopy.TableCache.Count * tableCacheSizeAdjustment
+	}
+	return redact.StringWithoutMarkers(&mCopy)
 }

vendor/github.com/cockroachdb/pebble/objstorage/objstorage.go

@@ -8,7 +8,10 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
+	"github.com/cockroachdb/pebble/objstorage/remote"
 	"github.com/cockroachdb/pebble/vfs"
 )
 
@@ -87,21 +90,71 @@ type ObjectMetadata struct {
 	DiskFileNum base.DiskFileNum
 	FileType    base.FileType
 
-	// The fields below are only set if the object is on shared storage.
+	// The fields below are only set if the object is on remote storage.
-	Shared struct {
+	Remote struct {
 		// CreatorID identifies the DB instance that originally created the object.
+		//
+		// Only used when CustomObjectName is not set.
 		CreatorID CreatorID
 		// CreatorFileNum is the identifier for the object within the context of the
 		// DB instance that originally created the object.
+		//
+		// Only used when CustomObjectName is not set.
 		CreatorFileNum base.DiskFileNum
-
+		// CustomObjectName (if it is set) overrides the object name that is normally
+		// derived from the CreatorID and CreatorFileNum.
+		CustomObjectName string
+		// CleanupMethod indicates the method for cleaning up unused shared objects.
 		CleanupMethod SharedCleanupMethod
+		// Locator identifies the remote.Storage implementation for this object.
+		Locator remote.Locator
+		// Storage is the remote.Storage object corresponding to the Locator. Used
+		// to avoid lookups in hot paths.
+		Storage remote.Storage
 	}
 }
 
-// IsShared returns true if the object is on shared storage.
+// IsRemote returns true if the object is on remote storage.
+func (meta *ObjectMetadata) IsRemote() bool {
+	return meta.IsShared() || meta.IsExternal()
+}
+
+// IsExternal returns true if the object is on remote storage but is not owned
+// by any Pebble instances in the cluster.
+func (meta *ObjectMetadata) IsExternal() bool {
+	return meta.Remote.CustomObjectName != ""
+}
+
+// IsShared returns true if the object is on remote storage and is owned by a
+// Pebble instance in the cluster (potentially shared between multiple
+// instances).
 func (meta *ObjectMetadata) IsShared() bool {
-	return meta.Shared.CreatorID.IsSet()
+	return meta.Remote.CreatorID.IsSet()
+}
+
+// AssertValid checks that the metadata is sane.
+func (meta *ObjectMetadata) AssertValid() {
+	if !meta.IsRemote() {
+		// Verify all Remote fields are empty.
+		if meta.Remote != (ObjectMetadata{}).Remote {
+			panic(errors.AssertionFailedf("meta.Remote not empty: %#v", meta.Remote))
+		}
+	} else {
+		if meta.Remote.CustomObjectName != "" {
+			if meta.Remote.CreatorID == 0 {
+				panic(errors.AssertionFailedf("CreatorID not set"))
+			}
+			if meta.Remote.CreatorFileNum == base.FileNum(0).DiskFileNum() {
+				panic(errors.AssertionFailedf("CreatorFileNum not set"))
+			}
+		}
+		if meta.Remote.CleanupMethod != SharedNoCleanup && meta.Remote.CleanupMethod != SharedRefTracking {
+			panic(errors.AssertionFailedf("invalid CleanupMethod %d", meta.Remote.CleanupMethod))
+		}
+		if meta.Remote.Storage == nil {
+			panic(errors.AssertionFailedf("Storage not set"))
+		}
+	}
 }
 
 // CreatorID identifies the DB instance that originally created a shared object.
@@ -122,7 +175,7 @@ const (
 	// keep track of references via reference marker objects.
 	SharedRefTracking SharedCleanupMethod = iota
 
-	// SharedNoCleanup is used for shared objects that are managed externally; the
+	// SharedNoCleanup is used for remote objects that are managed externally; the
 	// objstorage provider never deletes such objects.
 	SharedNoCleanup
 )
@@ -155,7 +208,7 @@ type CreateOptions struct {
 // created by the provider, or existing objects the Provider was informed about
 // via AddObjects.
 //
-// Objects are currently backed by a vfs.File or a shared.Storage object.
+// Objects are currently backed by a vfs.File or a remote.Storage object.
 type Provider interface {
 	// OpenForReading opens an existing object.
 	OpenForReading(
@@ -210,55 +263,62 @@ type Provider interface {
 	List() []ObjectMetadata
 
 	// SetCreatorID sets the CreatorID which is needed in order to use shared
-	// objects. Shared object usage is disabled until this method is called the
+	// objects. Remote object usage is disabled until this method is called the
 	// first time. Once set, the Creator ID is persisted and cannot change.
 	//
 	// Cannot be called if shared storage is not configured for the provider.
 	SetCreatorID(creatorID CreatorID) error
 
-	// IsForeign returns whether this object is owned by a different node. Return
+	// IsSharedForeign returns whether this object is owned by a different node.
-	// value undefined if creator ID is not set yet, or if this object does not
+	IsSharedForeign(meta ObjectMetadata) bool
-	// exist in this provider.
-	IsForeign(meta ObjectMetadata) bool
 
-	// SharedObjectBacking encodes the shared object metadata.
+	// RemoteObjectBacking encodes the remote object metadata for the given object.
-	SharedObjectBacking(meta *ObjectMetadata) (SharedObjectBackingHandle, error)
+	RemoteObjectBacking(meta *ObjectMetadata) (RemoteObjectBackingHandle, error)
 
-	// AttachSharedObjects registers existing shared objects with this provider.
+	// CreateExternalObjectBacking creates a backing for an existing object with a
-	AttachSharedObjects(objs []SharedObjectToAttach) ([]ObjectMetadata, error)
+	// custom object name. The object is considered to be managed outside of
+	// Pebble and will never be removed by Pebble.
+	CreateExternalObjectBacking(locator remote.Locator, objName string) (RemoteObjectBacking, error)
+
+	// AttachRemoteObjects registers existing remote objects with this provider.
+	AttachRemoteObjects(objs []RemoteObjectToAttach) ([]ObjectMetadata, error)
 
 	Close() error
 
 	// IsNotExistError indicates whether the error is known to report that a file or
 	// directory does not exist.
 	IsNotExistError(err error) bool
+
+	// Metrics returns metrics about objstorage. Currently, it only returns metrics
+	// about the shared cache.
+	Metrics() sharedcache.Metrics
 }
 
-// SharedObjectBacking encodes the metadata necessary to incorporate a shared
+// RemoteObjectBacking encodes the metadata necessary to incorporate a shared
 // object into a different Pebble instance. The encoding is specific to a given
 // Provider implementation.
-type SharedObjectBacking []byte
+type RemoteObjectBacking []byte
 
-// SharedObjectBackingHandle is a container for a SharedObjectBacking which
+// RemoteObjectBackingHandle is a container for a RemoteObjectBacking which
 // ensures that the backing stays valid. A backing can otherwise become invalid
-// if this provider unrefs the shared object. The SharedObjectBackingHandle
+// if this provider unrefs the shared object. The RemoteObjectBackingHandle
 // delays any unref until Close.
-type SharedObjectBackingHandle interface {
+type RemoteObjectBackingHandle interface {
 	// Get returns the backing. The backing is only guaranteed to be valid until
 	// Close is called (or until the Provider is closed). If Close was already
 	// called, returns an error.
-	Get() (SharedObjectBacking, error)
+	Get() (RemoteObjectBacking, error)
 	Close()
 }
 
-// SharedObjectToAttach contains the arguments needed to attach an existing shared object.
+// RemoteObjectToAttach contains the arguments needed to attach an existing remote object.
-type SharedObjectToAttach struct {
+type RemoteObjectToAttach struct {
 	// FileNum is the file number that will be used to refer to this object (in
 	// the context of this instance).
 	FileNum  base.DiskFileNum
 	FileType base.FileType
-	// Backing contains the metadata for the shared object backing (normally
+	// Backing contains the metadata for the remote object backing (normally
 	// generated from a different instance, but using the same Provider
 	// implementation).
-	Backing SharedObjectBacking
+	Backing RemoteObjectBacking
 }

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go

@@ -17,8 +17,9 @@ import (
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
-	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
-	"github.com/cockroachdb/pebble/objstorage/shared"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
+	"github.com/cockroachdb/pebble/objstorage/remote"
 	"github.com/cockroachdb/pebble/vfs"
 )
 
@@ -30,18 +31,20 @@ type provider struct {
 
 	tracer *objiotracing.Tracer
 
-	shared sharedSubsystem
+	remote remoteSubsystem
 
 	mu struct {
 		sync.RWMutex
 
-		shared struct {
+		remote struct {
-			// catalogBatch accumulates shared object creations and deletions until
+			// catalogBatch accumulates remote object creations and deletions until
 			// Sync is called.
-			catalogBatch sharedobjcat.Batch
+			catalogBatch remoteobjcat.Batch
+
+			storageObjects map[remote.Locator]remote.Storage
 		}
 
-		// localObjectsChanged is set if non-shared objects were created or deleted
+		// localObjectsChanged is set if non-remote objects were created or deleted
 		// but Sync was not yet called.
 		localObjectsChanged bool
 
@@ -89,18 +92,44 @@ type Settings struct {
 	// out a large chunk of dirty filesystem buffers.
 	BytesPerSync int
 
-	// Fields here are set only if the provider is to support shared objects
+	// Local contains fields that are only relevant for files stored on the local
+	// filesystem.
+	Local struct {
+		// TODO(radu): move FSCleaner, NoSyncOnClose, BytesPerSync here.
+
+		// ReadaheadConfigFn is a function used to retrieve the current readahead
+		// mode. This function is run whenever a local object is open for reading.
+		// If it is nil, DefaultReadaheadConfig is used.
+		ReadaheadConfigFn func() ReadaheadConfig
+	}
+
+	// Fields here are set only if the provider is to support remote objects
 	// (experimental).
-	Shared struct {
+	Remote struct {
-		Storage shared.Storage
+		StorageFactory remote.StorageFactory
+
+		// If CreateOnShared is non-zero, sstables are created on remote storage using
+		// the CreateOnSharedLocator (when the PreferSharedStorage create option is
+		// true).
+		CreateOnShared        remote.CreateOnSharedStrategy
+		CreateOnSharedLocator remote.Locator
 
 		// CacheSizeBytes is the size of the on-disk block cache for objects
-		// on shared storage. If it is 0, no cache is used.
+		// on remote storage. If it is 0, no cache is used.
 		CacheSizeBytes int64
 
 		// CacheBlockSize is the block size of the cache; if 0, the default of 32KB is used.
 		CacheBlockSize int
 
+		// ShardingBlockSize is the size of a shard block. The cache is split into contiguous
+		// ShardingBlockSize units. The units are distributed across multiple independent shards
+		// of the cache, via a hash(offset) modulo num shards operation. The cache replacement
+		// policies operate at the level of shard, not whole cache. This is done to reduce lock
+		// contention.
+		//
+		// If ShardingBlockSize is 0, the default of 1 MB is used.
+		ShardingBlockSize int64
+
 		// The number of independent shards the cache leverages. Each shard is the same size,
 		// and a hash of filenum & offset map a read to a certain shard. If set to 0,
 		// 2*runtime.GOMAXPROCS is used as the shard count.
@@ -111,7 +140,44 @@ type Settings struct {
 	}
 }
 
-// DefaultSettings initializes default settings (with no shared storage),
+// ReadaheadConfig controls the use of read-ahead.
+type ReadaheadConfig struct {
+	// Informed is the type of read-ahead for operations that are known to read a
+	// large consecutive chunk of a file.
+	Informed ReadaheadMode
+
+	// Speculative is the type of read-ahead used automatically, when consecutive
+	// reads are detected.
+	Speculative ReadaheadMode
+}
+
+// DefaultReadaheadConfig is the readahead config used when ReadaheadConfigFn is
+// not specified.
+var DefaultReadaheadConfig = ReadaheadConfig{
+	Informed:    FadviseSequential,
+	Speculative: FadviseSequential,
+}
+
+// ReadaheadMode indicates the type of read-ahead to use, either for informed
+// read-ahead (e.g. compactions) or speculative read-ahead.
+type ReadaheadMode uint8
+
+const (
+	// NoReadahead disables readahead altogether.
+	NoReadahead ReadaheadMode = iota
+
+	// SysReadahead enables the use of SYS_READAHEAD call to prefetch data.
+	// The prefetch window grows dynamically as consecutive writes are detected.
+	SysReadahead
+
+	// FadviseSequential enables to use of FADV_SEQUENTIAL. For informed
+	// read-ahead, FADV_SEQUENTIAL is used from the beginning. For speculative
+	// read-ahead SYS_READAHEAD is first used until the window reaches the maximum
+	// size, then we siwtch to FADV_SEQUENTIAL.
+	FadviseSequential
+)
+
+// DefaultSettings initializes default settings (with no remote storage),
 // suitable for tests and tools.
 func DefaultSettings(fs vfs.FS, dirName string) Settings {
 	return Settings{
@@ -126,7 +192,13 @@ func DefaultSettings(fs vfs.FS, dirName string) Settings {
 
 // Open creates the provider.
 func Open(settings Settings) (objstorage.Provider, error) {
-	return open(settings)
+	// Note: we can't just `return open(settings)` because in an error case we
+	// would return (*provider)(nil) which is not objstorage.Provider(nil).
+	p, err := open(settings)
+	if err != nil {
+		return nil, err
+	}
+	return p, nil
 }
 
 func open(settings Settings) (p *provider, _ error) {
@@ -157,8 +229,8 @@ func open(settings Settings) (p *provider, _ error) {
 		return nil, err
 	}
 
-	// Initialize shared subsystem (if configured) and add shared objects.
+	// Initialize remote subsystem (if configured) and add remote objects.
-	if err := p.sharedInit(); err != nil {
+	if err := p.remoteInit(); err != nil {
 		return nil, err
 	}
 
@@ -167,9 +239,9 @@ func open(settings Settings) (p *provider, _ error) {
 
 // Close is part of the objstorage.Provider interface.
 func (p *provider) Close() error {
-	var err error
+	err := p.sharedClose()
 	if p.fsDir != nil {
-		err = p.fsDir.Close()
+		err = firstError(err, p.fsDir.Close())
 		p.fsDir = nil
 	}
 	if objiotracing.Enabled {
@@ -197,10 +269,14 @@ func (p *provider) OpenForReading(
 	}
 
 	var r objstorage.Readable
-	if !meta.IsShared() {
+	if !meta.IsRemote() {
 		r, err = p.vfsOpenForReading(ctx, fileType, fileNum, opts)
 	} else {
-		r, err = p.sharedOpenForReading(ctx, meta, opts)
+		r, err = p.remoteOpenForReading(ctx, meta, opts)
+		if err != nil && p.isNotExistError(meta, err) {
+			// Wrap the error so that IsNotExistError functions properly.
+			err = errors.Mark(err, os.ErrNotExist)
+		}
 	}
 	if err != nil {
 		return nil, err
@@ -221,8 +297,8 @@ func (p *provider) Create(
 	fileNum base.DiskFileNum,
 	opts objstorage.CreateOptions,
 ) (w objstorage.Writable, meta objstorage.ObjectMetadata, err error) {
-	if opts.PreferSharedStorage && p.st.Shared.Storage != nil {
+	if opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone {
-		w, meta, err = p.sharedCreate(ctx, fileType, fileNum, opts)
+		w, meta, err = p.sharedCreate(ctx, fileType, fileNum, p.st.Remote.CreateOnSharedLocator, opts)
 	} else {
 		w, meta, err = p.vfsCreate(ctx, fileType, fileNum)
 	}
@@ -239,8 +315,8 @@ func (p *provider) Create(
 
 // Remove removes an object.
 //
-// Note that if the object is shared, the object is only (conceptually) removed
+// Note that if the object is remote, the object is only (conceptually) removed
-// from this provider. If other providers have references on the shared object,
+// from this provider. If other providers have references on the remote object,
 // it will not be removed.
 //
 // The object is not guaranteed to be durably removed until Sync is called.
@@ -250,11 +326,15 @@ func (p *provider) Remove(fileType base.FileType, fileNum base.DiskFileNum) erro
 		return err
 	}
 
-	if !meta.IsShared() {
+	if !meta.IsRemote() {
 		err = p.vfsRemove(fileType, fileNum)
 	} else {
-		// TODO(radu): implement shared object removal (i.e. deref).
+		// TODO(radu): implement remote object removal (i.e. deref).
 		err = p.sharedUnref(meta)
+		if err != nil && p.isNotExistError(meta, err) {
+			// Wrap the error so that IsNotExistError functions properly.
+			err = errors.Mark(err, os.ErrNotExist)
+		}
 	}
 	if err != nil && !p.IsNotExistError(err) {
 		// We want to be able to retry a Remove, so we keep the object in our list.
@@ -267,15 +347,18 @@ func (p *provider) Remove(fileType base.FileType, fileNum base.DiskFileNum) erro
 	return err
 }
 
+func (p *provider) isNotExistError(meta objstorage.ObjectMetadata, err error) bool {
+	if meta.Remote.Storage != nil {
+		return meta.Remote.Storage.IsNotExistError(err)
+	}
+	return oserror.IsNotExist(err)
+}
+
 // IsNotExistError is part of the objstorage.Provider interface.
 func (p *provider) IsNotExistError(err error) bool {
-	if oserror.IsNotExist(err) {
+	// We use errors.Mark(err, os.ErrNotExist) for not-exist errors coming from
-		return true
+	// remote.Storage.
-	}
+	return oserror.IsNotExist(err)
-	if p.sharedStorage() != nil && p.sharedStorage().IsNotExistError(err) {
-		return true
-	}
-	return false
 }
 
 // Sync flushes the metadata from creation or removal of objects since the last Sync.
@@ -303,7 +386,7 @@ func (p *provider) LinkOrCopyFromLocal(
 	dstFileNum base.DiskFileNum,
 	opts objstorage.CreateOptions,
 ) (objstorage.ObjectMetadata, error) {
-	shared := opts.PreferSharedStorage && p.st.Shared.Storage != nil
+	shared := opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone
 	if !shared && srcFS == p.st.FS {
 		// Wrap the normal filesystem with one which wraps newly created files with
 		// vfs.NewSyncingFile.
@@ -383,18 +466,18 @@ func (p *provider) Lookup(
 
 // Path is part of the objstorage.Provider interface.
 func (p *provider) Path(meta objstorage.ObjectMetadata) string {
-	if !meta.IsShared() {
+	if !meta.IsRemote() {
 		return p.vfsPath(meta.FileType, meta.DiskFileNum)
 	}
-	return p.sharedPath(meta)
+	return p.remotePath(meta)
 }
 
 // Size returns the size of the object.
 func (p *provider) Size(meta objstorage.ObjectMetadata) (int64, error) {
-	if !meta.IsShared() {
+	if !meta.IsRemote() {
 		return p.vfsSize(meta.FileType, meta.DiskFileNum)
 	}
-	return p.sharedSize(meta)
+	return p.remoteSize(meta)
 }
 
 // List is part of the objstorage.Provider interface.
@@ -411,17 +494,29 @@ func (p *provider) List() []objstorage.ObjectMetadata {
 	return res
 }
 
+// Metrics is part of the objstorage.Provider interface.
+func (p *provider) Metrics() sharedcache.Metrics {
+	if p.remote.cache != nil {
+		return p.remote.cache.Metrics()
+	}
+	return sharedcache.Metrics{}
+}
+
 func (p *provider) addMetadata(meta objstorage.ObjectMetadata) {
+	if invariants.Enabled {
+		meta.AssertValid()
+	}
 	p.mu.Lock()
 	defer p.mu.Unlock()
 	p.mu.knownObjects[meta.DiskFileNum] = meta
-	if meta.IsShared() {
+	if meta.IsRemote() {
-		p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
+		p.mu.remote.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
 			FileNum:        meta.DiskFileNum,
 			FileType:       meta.FileType,
-			CreatorID:      meta.Shared.CreatorID,
+			CreatorID:      meta.Remote.CreatorID,
-			CreatorFileNum: meta.Shared.CreatorFileNum,
+			CreatorFileNum: meta.Remote.CreatorFileNum,
-			CleanupMethod:  meta.Shared.CleanupMethod,
+			Locator:        meta.Remote.Locator,
+			CleanupMethod:  meta.Remote.CleanupMethod,
 		})
 	} else {
 		p.mu.localObjectsChanged = true
@@ -437,14 +532,14 @@ func (p *provider) removeMetadata(fileNum base.DiskFileNum) {
 		return
 	}
 	delete(p.mu.knownObjects, fileNum)
-	if meta.IsShared() {
+	if meta.IsRemote() {
-		p.mu.shared.catalogBatch.DeleteObject(fileNum)
+		p.mu.remote.catalogBatch.DeleteObject(fileNum)
 	} else {
 		p.mu.localObjectsChanged = true
 	}
 }
 
-// protectObject prevents the unreferencing of a shared object until
+// protectObject prevents the unreferencing of a remote object until
 // unprotectObject is called.
 func (p *provider) protectObject(fileNum base.DiskFileNum) {
 	p.mu.Lock()

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go

@@ -4,6 +4,8 @@
 
 package objstorageprovider
 
+import "github.com/cockroachdb/pebble/internal/invariants"
+
 const (
 	// Constants for dynamic readahead of data blocks. Note that the size values
 	// make sense as some multiple of the default block size; and they should
@@ -81,6 +83,9 @@ func (rs *readaheadState) recordCacheHit(offset, blockLength int64) {
 // Returns a size value (greater than 0) that should be prefetched if readahead
 // would be beneficial.
 func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 {
+	if invariants.Enabled && rs.maxReadaheadSize == 0 {
+		panic("readaheadState not initialized")
+	}
 	currentReadEnd := offset + blockLength
 	if rs.numReads >= minFileReadsForReadahead {
 		// The minimum threshold of sequential reads to justify reading ahead

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote.go

@@ -0,0 +1,376 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"sync"
+	"sync/atomic"
+
+	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
+	"github.com/cockroachdb/pebble/objstorage/remote"
+)
+
+// remoteSubsystem contains the provider fields related to remote storage.
+// All fields remain unset if remote storage is not configured.
+type remoteSubsystem struct {
+	catalog *remoteobjcat.Catalog
+	// catalogSyncMutex is used to correctly serialize two sharedSync operations.
+	// It must be acquired before the provider mutex.
+	catalogSyncMutex sync.Mutex
+
+	cache *sharedcache.Cache
+
+	// shared contains the fields relevant to shared objects, i.e. objects that
+	// are created by Pebble and potentially shared between Pebble instances.
+	shared struct {
+		// initialized guards access to the creatorID field.
+		initialized atomic.Bool
+		creatorID   objstorage.CreatorID
+		initOnce    sync.Once
+
+		// checkRefsOnOpen controls whether we check the ref marker file when opening
+		// an object. Normally this is true when invariants are enabled (but the provider
+		// test tweaks this field).
+		checkRefsOnOpen bool
+	}
+}
+
+// remoteInit initializes the remote object subsystem (if configured) and finds
+// any remote objects.
+func (p *provider) remoteInit() error {
+	if p.st.Remote.StorageFactory == nil {
+		return nil
+	}
+	catalog, contents, err := remoteobjcat.Open(p.st.FS, p.st.FSDirName)
+	if err != nil {
+		return errors.Wrapf(err, "pebble: could not open remote object catalog")
+	}
+	p.remote.catalog = catalog
+	p.remote.shared.checkRefsOnOpen = invariants.Enabled
+
+	// The creator ID may or may not be initialized yet.
+	if contents.CreatorID.IsSet() {
+		p.remote.initShared(contents.CreatorID)
+		p.st.Logger.Infof("remote storage configured; creatorID = %s", contents.CreatorID)
+	} else {
+		p.st.Logger.Infof("remote storage configured; no creatorID yet")
+	}
+
+	if p.st.Remote.CacheSizeBytes > 0 {
+		const defaultBlockSize = 32 * 1024
+		blockSize := p.st.Remote.CacheBlockSize
+		if blockSize == 0 {
+			blockSize = defaultBlockSize
+		}
+
+		const defaultShardingBlockSize = 1024 * 1024
+		shardingBlockSize := p.st.Remote.ShardingBlockSize
+		if shardingBlockSize == 0 {
+			shardingBlockSize = defaultShardingBlockSize
+		}
+
+		numShards := p.st.Remote.CacheShardCount
+		if numShards == 0 {
+			numShards = 2 * runtime.GOMAXPROCS(0)
+		}
+
+		p.remote.cache, err = sharedcache.Open(
+			p.st.FS, p.st.Logger, p.st.FSDirName, blockSize, shardingBlockSize, p.st.Remote.CacheSizeBytes, numShards)
+		if err != nil {
+			return errors.Wrapf(err, "pebble: could not open remote object cache")
+		}
+	}
+
+	for _, meta := range contents.Objects {
+		o := objstorage.ObjectMetadata{
+			DiskFileNum: meta.FileNum,
+			FileType:    meta.FileType,
+		}
+		o.Remote.CreatorID = meta.CreatorID
+		o.Remote.CreatorFileNum = meta.CreatorFileNum
+		o.Remote.CleanupMethod = meta.CleanupMethod
+		o.Remote.Locator = meta.Locator
+		o.Remote.CustomObjectName = meta.CustomObjectName
+		o.Remote.Storage, err = p.ensureStorageLocked(o.Remote.Locator)
+		if err != nil {
+			return errors.Wrapf(err, "creating remote.Storage object for locator '%s'", o.Remote.Locator)
+		}
+		if invariants.Enabled {
+			o.AssertValid()
+		}
+		p.mu.knownObjects[o.DiskFileNum] = o
+	}
+	return nil
+}
+
+// initShared initializes the creator ID, allowing use of shared objects.
+func (ss *remoteSubsystem) initShared(creatorID objstorage.CreatorID) {
+	ss.shared.initOnce.Do(func() {
+		ss.shared.creatorID = creatorID
+		ss.shared.initialized.Store(true)
+	})
+}
+
+func (p *provider) sharedClose() error {
+	if p.st.Remote.StorageFactory == nil {
+		return nil
+	}
+	var err error
+	if p.remote.cache != nil {
+		err = p.remote.cache.Close()
+		p.remote.cache = nil
+	}
+	if p.remote.catalog != nil {
+		err = firstError(err, p.remote.catalog.Close())
+		p.remote.catalog = nil
+	}
+	return err
+}
+
+// SetCreatorID is part of the objstorage.Provider interface.
+func (p *provider) SetCreatorID(creatorID objstorage.CreatorID) error {
+	if p.st.Remote.StorageFactory == nil {
+		return errors.AssertionFailedf("attempt to set CreatorID but remote storage not enabled")
+	}
+	// Note: this call is a cheap no-op if the creator ID was already set. This
+	// call also checks if we are trying to change the ID.
+	if err := p.remote.catalog.SetCreatorID(creatorID); err != nil {
+		return err
+	}
+	if !p.remote.shared.initialized.Load() {
+		p.st.Logger.Infof("remote storage creatorID set to %s", creatorID)
+		p.remote.initShared(creatorID)
+	}
+	return nil
+}
+
+// IsSharedForeign is part of the objstorage.Provider interface.
+func (p *provider) IsSharedForeign(meta objstorage.ObjectMetadata) bool {
+	if !p.remote.shared.initialized.Load() {
+		return false
+	}
+	return meta.IsShared() && (meta.Remote.CreatorID != p.remote.shared.creatorID)
+}
+
+func (p *provider) remoteCheckInitialized() error {
+	if p.st.Remote.StorageFactory == nil {
+		return errors.Errorf("remote object support not configured")
+	}
+	return nil
+}
+
+func (p *provider) sharedCheckInitialized() error {
+	if err := p.remoteCheckInitialized(); err != nil {
+		return err
+	}
+	if !p.remote.shared.initialized.Load() {
+		return errors.Errorf("remote object support not available: remote creator ID not yet set")
+	}
+	return nil
+}
+
+func (p *provider) sharedSync() error {
+	// Serialize parallel sync operations. Note that ApplyBatch is already
+	// serialized internally, but we want to make sure they get called with
+	// batches in the right order.
+	p.remote.catalogSyncMutex.Lock()
+	defer p.remote.catalogSyncMutex.Unlock()
+
+	batch := func() remoteobjcat.Batch {
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		res := p.mu.remote.catalogBatch.Copy()
+		p.mu.remote.catalogBatch.Reset()
+		return res
+	}()
+
+	if batch.IsEmpty() {
+		return nil
+	}
+
+	if err := p.remote.catalog.ApplyBatch(batch); err != nil {
+		// Put back the batch (for the next Sync), appending any operations that
+		// happened in the meantime.
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		batch.Append(p.mu.remote.catalogBatch)
+		p.mu.remote.catalogBatch = batch
+		return err
+	}
+
+	return nil
+}
+
+func (p *provider) remotePath(meta objstorage.ObjectMetadata) string {
+	if meta.Remote.Locator != "" {
+		return fmt.Sprintf("remote-%s://%s", meta.Remote.Locator, remoteObjectName(meta))
+	}
+	return "remote://" + remoteObjectName(meta)
+}
+
+// sharedCreateRef creates a reference marker object.
+func (p *provider) sharedCreateRef(meta objstorage.ObjectMetadata) error {
+	if err := p.sharedCheckInitialized(); err != nil {
+		return err
+	}
+	if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
+		return nil
+	}
+	refName := p.sharedObjectRefName(meta)
+	writer, err := meta.Remote.Storage.CreateObject(refName)
+	if err == nil {
+		// The object is empty, just close the writer.
+		err = writer.Close()
+	}
+	if err != nil {
+		return errors.Wrapf(err, "creating marker object %q", refName)
+	}
+	return nil
+}
+
+func (p *provider) sharedCreate(
+	_ context.Context,
+	fileType base.FileType,
+	fileNum base.DiskFileNum,
+	locator remote.Locator,
+	opts objstorage.CreateOptions,
+) (objstorage.Writable, objstorage.ObjectMetadata, error) {
+	if err := p.sharedCheckInitialized(); err != nil {
+		return nil, objstorage.ObjectMetadata{}, err
+	}
+	storage, err := p.ensureStorage(locator)
+	if err != nil {
+		return nil, objstorage.ObjectMetadata{}, err
+	}
+	meta := objstorage.ObjectMetadata{
+		DiskFileNum: fileNum,
+		FileType:    fileType,
+	}
+	meta.Remote.CreatorID = p.remote.shared.creatorID
+	meta.Remote.CreatorFileNum = fileNum
+	meta.Remote.CleanupMethod = opts.SharedCleanupMethod
+	meta.Remote.Locator = locator
+	meta.Remote.Storage = storage
+
+	objName := remoteObjectName(meta)
+	writer, err := storage.CreateObject(objName)
+	if err != nil {
+		return nil, objstorage.ObjectMetadata{}, errors.Wrapf(err, "creating object %q", objName)
+	}
+	return &sharedWritable{
+		p:             p,
+		meta:          meta,
+		storageWriter: writer,
+	}, meta, nil
+}
+
+func (p *provider) remoteOpenForReading(
+	ctx context.Context, meta objstorage.ObjectMetadata, opts objstorage.OpenOptions,
+) (objstorage.Readable, error) {
+	if err := p.remoteCheckInitialized(); err != nil {
+		return nil, err
+	}
+	// Verify we have a reference on this object; for performance reasons, we only
+	// do this in testing scenarios.
+	if p.remote.shared.checkRefsOnOpen && meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
+		if err := p.sharedCheckInitialized(); err != nil {
+			return nil, err
+		}
+		refName := p.sharedObjectRefName(meta)
+		if _, err := meta.Remote.Storage.Size(refName); err != nil {
+			if meta.Remote.Storage.IsNotExistError(err) {
+				if opts.MustExist {
+					p.st.Logger.Fatalf("marker object %q does not exist", refName)
+					// TODO(radu): maybe list references for the object.
+				}
+				return nil, errors.Errorf("marker object %q does not exist", refName)
+			}
+			return nil, errors.Wrapf(err, "checking marker object %q", refName)
+		}
+	}
+	objName := remoteObjectName(meta)
+	reader, size, err := meta.Remote.Storage.ReadObject(ctx, objName)
+	if err != nil {
+		if opts.MustExist && meta.Remote.Storage.IsNotExistError(err) {
+			p.st.Logger.Fatalf("object %q does not exist", objName)
+			// TODO(radu): maybe list references for the object.
+		}
+		return nil, err
+	}
+	return p.newRemoteReadable(reader, size, meta.DiskFileNum), nil
+}
+
+func (p *provider) remoteSize(meta objstorage.ObjectMetadata) (int64, error) {
+	if err := p.remoteCheckInitialized(); err != nil {
+		return 0, err
+	}
+	objName := remoteObjectName(meta)
+	return meta.Remote.Storage.Size(objName)
+}
+
+// sharedUnref implements object "removal" with the remote backend. The ref
+// marker object is removed and the backing object is removed only if there are
+// no other ref markers.
+func (p *provider) sharedUnref(meta objstorage.ObjectMetadata) error {
+	if meta.Remote.CleanupMethod == objstorage.SharedNoCleanup {
+		// Never delete objects in this mode.
+		return nil
+	}
+	if p.isProtected(meta.DiskFileNum) {
+		// TODO(radu): we need a mechanism to unref the object when it becomes
+		// unprotected.
+		return nil
+	}
+
+	refName := p.sharedObjectRefName(meta)
+	// Tolerate a not-exists error.
+	if err := meta.Remote.Storage.Delete(refName); err != nil && !meta.Remote.Storage.IsNotExistError(err) {
+		return err
+	}
+	otherRefs, err := meta.Remote.Storage.List(sharedObjectRefPrefix(meta), "" /* delimiter */)
+	if err != nil {
+		return err
+	}
+	if len(otherRefs) == 0 {
+		objName := remoteObjectName(meta)
+		if err := meta.Remote.Storage.Delete(objName); err != nil && !meta.Remote.Storage.IsNotExistError(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+// ensureStorageLocked populates the remote.Storage object for the given
+// locator, if necessary. p.mu must be held.
+func (p *provider) ensureStorageLocked(locator remote.Locator) (remote.Storage, error) {
+	if p.mu.remote.storageObjects == nil {
+		p.mu.remote.storageObjects = make(map[remote.Locator]remote.Storage)
+	}
+	if res, ok := p.mu.remote.storageObjects[locator]; ok {
+		return res, nil
+	}
+	res, err := p.st.Remote.StorageFactory.CreateStorage(locator)
+	if err != nil {
+		return nil, err
+	}
+
+	p.mu.remote.storageObjects[locator] = res
+	return res, nil
+}
+
+// ensureStorage populates the remote.Storage object for the given locator, if necessary.
+func (p *provider) ensureStorage(locator remote.Locator) (remote.Storage, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.ensureStorageLocked(locator)
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_backing.go

@@ -12,7 +12,8 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/objstorage"
-	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
+	"github.com/cockroachdb/pebble/objstorage/remote"
 )
 
 const (
@@ -25,6 +26,12 @@ const (
 	// allows the "target" provider to check that the "source" provider kept its
 	// reference on the object alive.
 	tagRefCheckID = 4
+	// tagLocator encodes the remote.Locator; if absent the locator is "". It is
+	// followed by the locator string length and the locator string.
+	tagLocator = 5
+	// tagLocator encodes a custom object name (if present). It is followed by the
+	// custom name string length and the string.
+	tagCustomObjectName = 6
 
 	// Any new tags that don't have the tagNotSafeToIgnoreMask bit set must be
 	// followed by the length of the data (so they can be skipped).
@@ -34,83 +41,103 @@ const (
 	tagNotSafeToIgnoreMask = 64
 )
 
-func (p *provider) encodeSharedObjectBacking(
+func (p *provider) encodeRemoteObjectBacking(
 	meta *objstorage.ObjectMetadata,
-) (objstorage.SharedObjectBacking, error) {
+) (objstorage.RemoteObjectBacking, error) {
-	if !meta.IsShared() {
+	if !meta.IsRemote() {
-		return nil, errors.AssertionFailedf("object %s not on shared storage", meta.DiskFileNum)
+		return nil, errors.AssertionFailedf("object %s not on remote storage", meta.DiskFileNum)
 	}
 
 	buf := make([]byte, 0, binary.MaxVarintLen64*4)
 	buf = binary.AppendUvarint(buf, tagCreatorID)
-	buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorID))
+	buf = binary.AppendUvarint(buf, uint64(meta.Remote.CreatorID))
 	// TODO(radu): encode file type as well?
 	buf = binary.AppendUvarint(buf, tagCreatorFileNum)
-	buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorFileNum.FileNum()))
+	buf = binary.AppendUvarint(buf, uint64(meta.Remote.CreatorFileNum.FileNum()))
 	buf = binary.AppendUvarint(buf, tagCleanupMethod)
-	buf = binary.AppendUvarint(buf, uint64(meta.Shared.CleanupMethod))
+	buf = binary.AppendUvarint(buf, uint64(meta.Remote.CleanupMethod))
-	if meta.Shared.CleanupMethod == objstorage.SharedRefTracking {
+	if meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
 		buf = binary.AppendUvarint(buf, tagRefCheckID)
-		buf = binary.AppendUvarint(buf, uint64(p.shared.creatorID))
+		buf = binary.AppendUvarint(buf, uint64(p.remote.shared.creatorID))
 		buf = binary.AppendUvarint(buf, uint64(meta.DiskFileNum.FileNum()))
 	}
+	if meta.Remote.Locator != "" {
+		buf = binary.AppendUvarint(buf, tagLocator)
+		buf = encodeString(buf, string(meta.Remote.Locator))
+	}
+	if meta.Remote.CustomObjectName != "" {
+		buf = binary.AppendUvarint(buf, tagCustomObjectName)
+		buf = encodeString(buf, meta.Remote.CustomObjectName)
+	}
 	return buf, nil
 }
 
-type sharedObjectBackingHandle struct {
+type remoteObjectBackingHandle struct {
-	backing objstorage.SharedObjectBacking
+	backing objstorage.RemoteObjectBacking
 	fileNum base.DiskFileNum
 	p       *provider
 }
 
-func (s *sharedObjectBackingHandle) Get() (objstorage.SharedObjectBacking, error) {
+func (s *remoteObjectBackingHandle) Get() (objstorage.RemoteObjectBacking, error) {
 	if s.backing == nil {
-		return nil, errors.Errorf("SharedObjectBackingHandle.Get() called after Close()")
+		return nil, errors.Errorf("RemoteObjectBackingHandle.Get() called after Close()")
 	}
 	return s.backing, nil
 }
 
-func (s *sharedObjectBackingHandle) Close() {
+func (s *remoteObjectBackingHandle) Close() {
 	if s.backing != nil {
 		s.backing = nil
 		s.p.unprotectObject(s.fileNum)
 	}
 }
 
-var _ objstorage.SharedObjectBackingHandle = (*sharedObjectBackingHandle)(nil)
+var _ objstorage.RemoteObjectBackingHandle = (*remoteObjectBackingHandle)(nil)
 
-// SharedObjectBacking is part of the objstorage.Provider interface.
+// RemoteObjectBacking is part of the objstorage.Provider interface.
-func (p *provider) SharedObjectBacking(
+func (p *provider) RemoteObjectBacking(
 	meta *objstorage.ObjectMetadata,
-) (objstorage.SharedObjectBackingHandle, error) {
+) (objstorage.RemoteObjectBackingHandle, error) {
-	backing, err := p.encodeSharedObjectBacking(meta)
+	backing, err := p.encodeRemoteObjectBacking(meta)
 	if err != nil {
 		return nil, err
 	}
 	p.protectObject(meta.DiskFileNum)
-	return &sharedObjectBackingHandle{
+	return &remoteObjectBackingHandle{
 		backing: backing,
 		fileNum: meta.DiskFileNum,
 		p:       p,
 	}, nil
 }
 
+// CreateExternalObjectBacking is part of the objstorage.Provider interface.
+func (p *provider) CreateExternalObjectBacking(
+	locator remote.Locator, objName string,
+) (objstorage.RemoteObjectBacking, error) {
+	var meta objstorage.ObjectMetadata
+	meta.Remote.Locator = locator
+	meta.Remote.CustomObjectName = objName
+	meta.Remote.CleanupMethod = objstorage.SharedNoCleanup
+	return p.encodeRemoteObjectBacking(&meta)
+}
+
 type decodedBacking struct {
 	meta objstorage.ObjectMetadata
-	// refToCheck is set only when meta.Shared.CleanupMethod is RefTracking
+	// refToCheck is set only when meta.Remote.CleanupMethod is RefTracking
 	refToCheck struct {
 		creatorID objstorage.CreatorID
 		fileNum   base.DiskFileNum
 	}
 }
 
-// decodeSharedObjectBacking decodes the shared object metadata.
+// decodeRemoteObjectBacking decodes the remote object metadata.
-// Returns the object metadata and (optionally) the creator ID of the provider
+//
-// that encoded the backing whose ref marker needs to be checked.
+// Note that the meta.Remote.Storage field is not set.
-func decodeSharedObjectBacking(
+func decodeRemoteObjectBacking(
-	fileType base.FileType, fileNum base.DiskFileNum, buf objstorage.SharedObjectBacking,
+	fileType base.FileType, fileNum base.DiskFileNum, buf objstorage.RemoteObjectBacking,
 ) (decodedBacking, error) {
 	var creatorID, creatorFileNum, cleanupMethod, refCheckCreatorID, refCheckFileNum uint64
+	var locator, customObjName string
 	br := bytes.NewReader(buf)
 	for {
 		tag, err := binary.ReadUvarint(br)
@@ -136,6 +163,12 @@ func decodeSharedObjectBacking(
 				refCheckFileNum, err = binary.ReadUvarint(br)
 			}
 
+		case tagLocator:
+			locator, err = decodeString(br)
+
+		case tagCustomObjectName:
+			customObjName, err = decodeString(br)
+
 		default:
 			// Ignore unknown tags, unless they're not safe to ignore.
 			if tag&tagNotSafeToIgnoreMask != 0 {
@@ -151,37 +184,65 @@ func decodeSharedObjectBacking(
 			return decodedBacking{}, err
 		}
 	}
-	if creatorID == 0 {
+	if customObjName == "" {
-		return decodedBacking{}, errors.Newf("shared object backing missing creator ID")
+		if creatorID == 0 {
-	}
+			return decodedBacking{}, errors.Newf("remote object backing missing creator ID")
-	if creatorFileNum == 0 {
+		}
-		return decodedBacking{}, errors.Newf("shared object backing missing creator file num")
+		if creatorFileNum == 0 {
+			return decodedBacking{}, errors.Newf("remote object backing missing creator file num")
+		}
 	}
 	var res decodedBacking
 	res.meta.DiskFileNum = fileNum
 	res.meta.FileType = fileType
-	res.meta.Shared.CreatorID = objstorage.CreatorID(creatorID)
+	res.meta.Remote.CreatorID = objstorage.CreatorID(creatorID)
-	res.meta.Shared.CreatorFileNum = base.FileNum(creatorFileNum).DiskFileNum()
+	res.meta.Remote.CreatorFileNum = base.FileNum(creatorFileNum).DiskFileNum()
-	res.meta.Shared.CleanupMethod = objstorage.SharedCleanupMethod(cleanupMethod)
+	res.meta.Remote.CleanupMethod = objstorage.SharedCleanupMethod(cleanupMethod)
-
+	if res.meta.Remote.CleanupMethod == objstorage.SharedRefTracking {
-	if res.meta.Shared.CleanupMethod == objstorage.SharedRefTracking {
 		if refCheckCreatorID == 0 || refCheckFileNum == 0 {
-			return decodedBacking{}, errors.Newf("shared object backing missing ref to check")
+			return decodedBacking{}, errors.Newf("remote object backing missing ref to check")
 		}
 		res.refToCheck.creatorID = objstorage.CreatorID(refCheckCreatorID)
 		res.refToCheck.fileNum = base.FileNum(refCheckFileNum).DiskFileNum()
 	}
+	res.meta.Remote.Locator = remote.Locator(locator)
+	res.meta.Remote.CustomObjectName = customObjName
 	return res, nil
 }
 
-// AttachSharedObjects is part of the objstorage.Provider interface.
+func encodeString(buf []byte, s string) []byte {
-func (p *provider) AttachSharedObjects(
+	buf = binary.AppendUvarint(buf, uint64(len(s)))
-	objs []objstorage.SharedObjectToAttach,
+	buf = append(buf, []byte(s)...)
+	return buf
+}
+
+func decodeString(br io.ByteReader) (string, error) {
+	length, err := binary.ReadUvarint(br)
+	if err != nil || length == 0 {
+		return "", err
+	}
+	buf := make([]byte, length)
+	for i := range buf {
+		buf[i], err = br.ReadByte()
+		if err != nil {
+			return "", err
+		}
+	}
+	return string(buf), nil
+}
+
+// AttachRemoteObjects is part of the objstorage.Provider interface.
+func (p *provider) AttachRemoteObjects(
+	objs []objstorage.RemoteObjectToAttach,
 ) ([]objstorage.ObjectMetadata, error) {
 	decoded := make([]decodedBacking, len(objs))
 	for i, o := range objs {
 		var err error
-		decoded[i], err = decodeSharedObjectBacking(o.FileType, o.FileNum, o.Backing)
+		decoded[i], err = decodeRemoteObjectBacking(o.FileType, o.FileNum, o.Backing)
+		if err != nil {
+			return nil, err
+		}
+		decoded[i].meta.Remote.Storage, err = p.ensureStorage(decoded[i].meta.Remote.Locator)
 		if err != nil {
 			return nil, err
 		}
@@ -190,7 +251,7 @@ func (p *provider) AttachSharedObjects(
 	// Create the reference marker objects.
 	// TODO(radu): parallelize this.
 	for _, d := range decoded {
-		if d.meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
+		if d.meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
 			continue
 		}
 		if err := p.sharedCreateRef(d.meta); err != nil {
@@ -199,10 +260,10 @@ func (p *provider) AttachSharedObjects(
 		}
 		// Check the "origin"'s reference.
 		refName := sharedObjectRefName(d.meta, d.refToCheck.creatorID, d.refToCheck.fileNum)
-		if _, err := p.sharedStorage().Size(refName); err != nil {
+		if _, err := d.meta.Remote.Storage.Size(refName); err != nil {
 			_ = p.sharedUnref(d.meta)
 			// TODO(radu): clean up references previously created in this loop.
-			if p.sharedStorage().IsNotExistError(err) {
+			if d.meta.Remote.Storage.IsNotExistError(err) {
 				return nil, errors.Errorf("origin marker object %q does not exist;"+
 					" object probably removed from the provider which created the backing", refName)
 			}
@@ -214,11 +275,13 @@ func (p *provider) AttachSharedObjects(
 		p.mu.Lock()
 		defer p.mu.Unlock()
 		for _, d := range decoded {
-			p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{
+			p.mu.remote.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
 				FileNum:        d.meta.DiskFileNum,
 				FileType:       d.meta.FileType,
-				CreatorID:      d.meta.Shared.CreatorID,
+				CreatorID:      d.meta.Remote.CreatorID,
-				CreatorFileNum: d.meta.Shared.CreatorFileNum,
+				CreatorFileNum: d.meta.Remote.CreatorFileNum,
+				CleanupMethod:  d.meta.Remote.CleanupMethod,
+				Locator:        d.meta.Remote.Locator,
 			})
 		}
 	}()

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_obj_name.go

@@ -11,16 +11,19 @@ import (
 	"github.com/cockroachdb/pebble/objstorage"
 )
 
-// sharedObjectName returns the name of an object on shared storage.
+// remoteObjectName returns the name of an object on remote storage.
 //
 // For sstables, the format is: <hash>-<creator-id>-<file-num>.sst
 // For example: 1a3f-2-000001.sst
-func sharedObjectName(meta objstorage.ObjectMetadata) string {
+func remoteObjectName(meta objstorage.ObjectMetadata) string {
+	if meta.Remote.CustomObjectName != "" {
+		return meta.Remote.CustomObjectName
+	}
 	switch meta.FileType {
 	case base.FileTypeTable:
 		return fmt.Sprintf(
 			"%04x-%d-%06d.sst",
-			objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(),
+			objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(),
 		)
 	}
 	panic("unknown FileType")
@@ -34,25 +37,33 @@ func sharedObjectName(meta objstorage.ObjectMetadata) string {
 func sharedObjectRefName(
 	meta objstorage.ObjectMetadata, refCreatorID objstorage.CreatorID, refFileNum base.DiskFileNum,
 ) string {
-	if meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
+	if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
 		panic("ref object used when ref tracking disabled")
 	}
+	if meta.Remote.CustomObjectName != "" {
+		return fmt.Sprintf(
+			"%s.ref.%d.%06d", meta.Remote.CustomObjectName, refCreatorID, refFileNum.FileNum(),
+		)
+	}
 	switch meta.FileType {
 	case base.FileTypeTable:
 		return fmt.Sprintf(
 			"%04x-%d-%06d.sst.ref.%d.%06d",
-			objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(), refCreatorID, refFileNum.FileNum(),
+			objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(), refCreatorID, refFileNum.FileNum(),
 		)
 	}
 	panic("unknown FileType")
 }
 
 func sharedObjectRefPrefix(meta objstorage.ObjectMetadata) string {
+	if meta.Remote.CustomObjectName != "" {
+		return meta.Remote.CustomObjectName + ".ref."
+	}
 	switch meta.FileType {
 	case base.FileTypeTable:
 		return fmt.Sprintf(
 			"%04x-%d-%06d.sst.ref.",
-			objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(),
+			objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(),
 		)
 	}
 	panic("unknown FileType")
@@ -64,10 +75,10 @@ func sharedObjectRefPrefix(meta objstorage.ObjectMetadata) string {
 //
 // For example: 1a3f-2-000001.sst.ref.5.000008
 func (p *provider) sharedObjectRefName(meta objstorage.ObjectMetadata) string {
-	if meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
+	if meta.Remote.CleanupMethod != objstorage.SharedRefTracking {
 		panic("ref object used when ref tracking disabled")
 	}
-	return sharedObjectRefName(meta, p.shared.creatorID, meta.DiskFileNum)
+	return sharedObjectRefName(meta, p.remote.shared.creatorID, meta.DiskFileNum)
 }
 
 // objHash returns a 16-bit hash value derived from the creator ID and creator
@@ -76,5 +87,5 @@ func (p *provider) sharedObjectRefName(meta objstorage.ObjectMetadata) string {
 func objHash(meta objstorage.ObjectMetadata) uint16 {
 	const prime1 = 7459
 	const prime2 = 17539
-	return uint16(uint64(meta.Shared.CreatorID)*prime1 + uint64(meta.Shared.CreatorFileNum.FileNum())*prime2)
+	return uint16(uint64(meta.Remote.CreatorID)*prime1 + uint64(meta.Remote.CreatorFileNum.FileNum())*prime2)
 }

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_readable.go

@@ -11,26 +11,26 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
-	"github.com/cockroachdb/pebble/objstorage/shared"
+	"github.com/cockroachdb/pebble/objstorage/remote"
 )
 
-const sharedMaxReadaheadSize = 1024 * 1024 /* 1MB */
+const remoteMaxReadaheadSize = 1024 * 1024 /* 1MB */
 
-// sharedReadable is a very simple implementation of Readable on top of the
+// remoteReadable is a very simple implementation of Readable on top of the
-// ReadCloser returned by shared.Storage.CreateObject.
+// ReadCloser returned by remote.Storage.CreateObject.
-type sharedReadable struct {
+type remoteReadable struct {
-	objReader shared.ObjectReader
+	objReader remote.ObjectReader
 	size      int64
 	fileNum   base.DiskFileNum
 	provider  *provider
 }
 
-var _ objstorage.Readable = (*sharedReadable)(nil)
+var _ objstorage.Readable = (*remoteReadable)(nil)
 
-func (p *provider) newSharedReadable(
+func (p *provider) newRemoteReadable(
-	objReader shared.ObjectReader, size int64, fileNum base.DiskFileNum,
+	objReader remote.ObjectReader, size int64, fileNum base.DiskFileNum,
-) *sharedReadable {
+) *remoteReadable {
-	return &sharedReadable{
+	return &remoteReadable{
 		objReader: objReader,
 		size:      size,
 		fileNum:   fileNum,
@@ -39,43 +39,43 @@ func (p *provider) newSharedReadable(
 }
 
 // ReadAt is part of the objstorage.Readable interface.
-func (r *sharedReadable) ReadAt(ctx context.Context, p []byte, offset int64) error {
+func (r *remoteReadable) ReadAt(ctx context.Context, p []byte, offset int64) error {
 	return r.readInternal(ctx, p, offset, false /* forCompaction */)
 }
 
 // readInternal performs a read for the object, using the cache when
 // appropriate.
-func (r *sharedReadable) readInternal(
+func (r *remoteReadable) readInternal(
 	ctx context.Context, p []byte, offset int64, forCompaction bool,
 ) error {
-	if cache := r.provider.shared.cache; cache != nil {
+	if cache := r.provider.remote.cache; cache != nil {
 		flags := sharedcache.ReadFlags{
 			// Don't add data to the cache if this read is for a compaction.
 			ReadOnly: forCompaction,
 		}
-		return r.provider.shared.cache.ReadAt(ctx, r.fileNum, p, offset, r.objReader, r.size, flags)
+		return r.provider.remote.cache.ReadAt(ctx, r.fileNum, p, offset, r.objReader, r.size, flags)
 	}
 	return r.objReader.ReadAt(ctx, p, offset)
 }
 
-func (r *sharedReadable) Close() error {
+func (r *remoteReadable) Close() error {
 	defer func() { r.objReader = nil }()
 	return r.objReader.Close()
 }
 
-func (r *sharedReadable) Size() int64 {
+func (r *remoteReadable) Size() int64 {
 	return r.size
 }
 
-func (r *sharedReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
+func (r *remoteReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
 	// TODO(radu): use a pool.
-	rh := &sharedReadHandle{readable: r}
+	rh := &remoteReadHandle{readable: r}
-	rh.readahead.state = makeReadaheadState(sharedMaxReadaheadSize)
+	rh.readahead.state = makeReadaheadState(remoteMaxReadaheadSize)
 	return rh
 }
 
-type sharedReadHandle struct {
+type remoteReadHandle struct {
-	readable  *sharedReadable
+	readable  *remoteReadable
 	readahead struct {
 		state  readaheadState
 		data   []byte
@@ -84,10 +84,10 @@ type sharedReadHandle struct {
 	forCompaction bool
 }
 
-var _ objstorage.ReadHandle = (*sharedReadHandle)(nil)
+var _ objstorage.ReadHandle = (*remoteReadHandle)(nil)
 
 // ReadAt is part of the objstorage.ReadHandle interface.
-func (r *sharedReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) error {
+func (r *remoteReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) error {
 	readaheadSize := r.maybeReadahead(offset, len(p))
 
 	// Check if we already have the data from a previous read-ahead.
@@ -135,27 +135,27 @@ func (r *sharedReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) e
 	return r.readable.readInternal(ctx, p, offset, r.forCompaction)
 }
 
-func (r *sharedReadHandle) maybeReadahead(offset int64, len int) int {
+func (r *remoteReadHandle) maybeReadahead(offset int64, len int) int {
 	if r.forCompaction {
-		return sharedMaxReadaheadSize
+		return remoteMaxReadaheadSize
 	}
 	return int(r.readahead.state.maybeReadahead(offset, int64(len)))
 }
 
 // Close is part of the objstorage.ReadHandle interface.
-func (r *sharedReadHandle) Close() error {
+func (r *remoteReadHandle) Close() error {
 	r.readable = nil
 	r.readahead.data = nil
 	return nil
 }
 
 // SetupForCompaction is part of the objstorage.ReadHandle interface.
-func (r *sharedReadHandle) SetupForCompaction() {
+func (r *remoteReadHandle) SetupForCompaction() {
 	r.forCompaction = true
 }
 
 // RecordCacheHit is part of the objstorage.ReadHandle interface.
-func (r *sharedReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
+func (r *remoteReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
 	if !r.forCompaction {
 		r.readahead.state.recordCacheHit(offset, size)
 	}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/catalog.go

@@ -2,7 +2,7 @@
 // of this source code is governed by a BSD-style license that can be found in
 // the LICENSE file.
 
-package sharedobjcat
+package remoteobjcat
 
 import (
 	"fmt"
@@ -13,15 +13,16 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/remote"
 	"github.com/cockroachdb/pebble/record"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/cockroachdb/pebble/vfs/atomicfs"
 )
 
-// Catalog is used to manage the on-disk shared object catalog.
+// Catalog is used to manage the on-disk remote object catalog.
 //
 // The catalog file is a log of records, where each record is an encoded
-// versionEdit.
+// VersionEdit.
 type Catalog struct {
 	fs      vfs.FS
 	dirname string
@@ -29,7 +30,7 @@ type Catalog struct {
 		sync.Mutex
 
 		creatorID objstorage.CreatorID
-		objects   map[base.DiskFileNum]SharedObjectMetadata
+		objects   map[base.DiskFileNum]RemoteObjectMetadata
 
 		marker *atomicfs.Marker
 
@@ -44,8 +45,8 @@ type Catalog struct {
 	}
 }
 
-// SharedObjectMetadata encapsulates the data stored in the catalog file for each object.
+// RemoteObjectMetadata encapsulates the data stored in the catalog file for each object.
-type SharedObjectMetadata struct {
+type RemoteObjectMetadata struct {
 	// FileNum is the identifier for the object within the context of a single DB
 	// instance.
 	FileNum base.DiskFileNum
@@ -56,24 +57,29 @@ type SharedObjectMetadata struct {
 	// CreatorFileNum is the identifier for the object within the context of the
 	// DB instance that originally created the object.
 	CreatorFileNum base.DiskFileNum
-
+	// CleanupMethod indicates the method for cleaning up unused shared objects.
 	CleanupMethod objstorage.SharedCleanupMethod
+	// Locator identifies a remote.Storage implementation.
+	Locator remote.Locator
+	// CustomObjectName (if it is set) overrides the object name that is normally
+	// derived from the CreatorID and CreatorFileNum.
+	CustomObjectName string
 }
 
 const (
-	catalogFilenameBase = "SHARED-CATALOG"
+	catalogFilenameBase = "REMOTE-OBJ-CATALOG"
-	catalogMarkerName   = "shared-catalog"
+	catalogMarkerName   = "remote-obj-catalog"
 
 	// We create a new file when the size exceeds 1MB (and some other conditions
 	// hold; see record.RotationHelper).
 	rotateFileSize = 1024 * 1024 // 1MB
 )
 
-// CatalogContents contains the shared objects in the catalog.
+// CatalogContents contains the remote objects in the catalog.
 type CatalogContents struct {
 	// CreatorID, if it is set.
 	CreatorID objstorage.CreatorID
-	Objects   []SharedObjectMetadata
+	Objects   []RemoteObjectMetadata
 }
 
 // Open creates a Catalog and loads any existing catalog file, returning the
@@ -83,7 +89,7 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
 		fs:      fs,
 		dirname: dirname,
 	}
-	c.mu.objects = make(map[base.DiskFileNum]SharedObjectMetadata)
+	c.mu.objects = make(map[base.DiskFileNum]RemoteObjectMetadata)
 
 	var err error
 	c.mu.marker, c.mu.catalogFilename, err = atomicfs.LocateMarker(fs, dirname, catalogMarkerName)
@@ -102,7 +108,7 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) {
 	}
 	res := CatalogContents{
 		CreatorID: c.mu.creatorID,
-		Objects:   make([]SharedObjectMetadata, 0, len(c.mu.objects)),
+		Objects:   make([]RemoteObjectMetadata, 0, len(c.mu.objects)),
 	}
 	for _, meta := range c.mu.objects {
 		res.Objects = append(res.Objects, meta)
@@ -130,9 +136,9 @@ func (c *Catalog) SetCreatorID(id objstorage.CreatorID) error {
 		return nil
 	}
 
-	ve := versionEdit{CreatorID: id}
+	ve := VersionEdit{CreatorID: id}
 	if err := c.writeToCatalogFileLocked(&ve); err != nil {
-		return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
+		return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
 	}
 	c.mu.creatorID = id
 	return nil
@@ -159,14 +165,14 @@ func (c *Catalog) closeCatalogFile() error {
 
 // Batch is used to perform multiple object additions/deletions at once.
 type Batch struct {
-	ve versionEdit
+	ve VersionEdit
 }
 
 // AddObject adds a new object to the batch.
 //
 // The given FileNum must be new - it must not match that of any object that was
 // ever in the catalog.
-func (b *Batch) AddObject(meta SharedObjectMetadata) {
+func (b *Batch) AddObject(meta RemoteObjectMetadata) {
 	b.ve.NewObjects = append(b.ve.NewObjects, meta)
 }
 
@@ -190,7 +196,7 @@ func (b *Batch) IsEmpty() bool {
 func (b *Batch) Copy() Batch {
 	var res Batch
 	if len(b.ve.NewObjects) > 0 {
-		res.ve.NewObjects = make([]SharedObjectMetadata, len(b.ve.NewObjects))
+		res.ve.NewObjects = make([]RemoteObjectMetadata, len(b.ve.NewObjects))
 		copy(res.ve.NewObjects, b.ve.NewObjects)
 	}
 	if len(b.ve.DeletedObjects) > 0 {
@@ -212,29 +218,40 @@ func (c *Catalog) ApplyBatch(b Batch) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	for _, n := range b.ve.DeletedObjects {
+	// Sanity checks.
-		if _, exists := c.mu.objects[n]; !exists {
+	toAdd := make(map[base.DiskFileNum]struct{}, len(b.ve.NewObjects))
-			return errors.AssertionFailedf("deleting non-existent object %s", n)
+	exists := func(n base.DiskFileNum) bool {
+		_, ok := c.mu.objects[n]
+		if !ok {
+			_, ok = toAdd[n]
 		}
+		return ok
 	}
 	for _, meta := range b.ve.NewObjects {
-		if _, exists := c.mu.objects[meta.FileNum]; exists {
+		if exists(meta.FileNum) {
 			return errors.AssertionFailedf("adding existing object %s", meta.FileNum)
 		}
+		toAdd[meta.FileNum] = struct{}{}
+	}
+	for _, n := range b.ve.DeletedObjects {
+		if !exists(n) {
+			return errors.AssertionFailedf("deleting non-existent object %s", n)
+		}
 	}
 
 	if err := c.writeToCatalogFileLocked(&b.ve); err != nil {
-		return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err)
+		return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err)
 	}
 
-	// Apply the batch to our current state.
+	// Add new objects before deleting any objects. This allows for cases where
-	for _, n := range b.ve.DeletedObjects {
+	// the same batch adds and deletes an object.
-		delete(c.mu.objects, n)
-	}
 	for _, meta := range b.ve.NewObjects {
 		c.mu.objects[meta.FileNum] = meta
 	}
-	b.Reset()
+	for _, n := range b.ve.DeletedObjects {
+		delete(c.mu.objects, n)
+	}
+
 	return nil
 }
 
@@ -243,7 +260,7 @@ func (c *Catalog) loadFromCatalogFile(filename string) error {
 	f, err := c.fs.Open(catalogPath)
 	if err != nil {
 		return errors.Wrapf(
-			err, "pebble: could not open shared object catalog file %q for DB %q",
+			err, "pebble: could not open remote object catalog file %q for DB %q",
 			errors.Safe(filename), c.dirname,
 		)
 	}
@@ -255,32 +272,26 @@ func (c *Catalog) loadFromCatalogFile(filename string) error {
 			break
 		}
 		if err != nil {
-			return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
+			return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
 				errors.Safe(filename))
 		}
-		var ve versionEdit
+		var ve VersionEdit
-		err = ve.Decode(r)
+		if err := ve.Decode(r); err != nil {
-		if err != nil {
+			return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
-			return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q",
 				errors.Safe(filename))
 		}
 		// Apply the version edit to the current state.
-		if ve.CreatorID.IsSet() {
+		if err := ve.Apply(&c.mu.creatorID, c.mu.objects); err != nil {
-			c.mu.creatorID = ve.CreatorID
+			return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q",
-		}
+				errors.Safe(filename))
-		for _, fileNum := range ve.DeletedObjects {
-			delete(c.mu.objects, fileNum)
-		}
-		for _, meta := range ve.NewObjects {
-			c.mu.objects[meta.FileNum] = meta
 		}
 	}
 	return nil
 }
 
-// writeToCatalogFileLocked writes a versionEdit to the catalog file.
+// writeToCatalogFileLocked writes a VersionEdit to the catalog file.
 // Creates a new file if this is the first write.
-func (c *Catalog) writeToCatalogFileLocked(ve *versionEdit) error {
+func (c *Catalog) writeToCatalogFileLocked(ve *VersionEdit) error {
 	c.mu.rotationHelper.AddRecord(int64(len(ve.NewObjects) + len(ve.DeletedObjects)))
 	snapshotSize := int64(len(c.mu.objects))
 
@@ -323,10 +334,10 @@ func (c *Catalog) createNewCatalogFileLocked() (outErr error) {
 	}
 	recWriter := record.NewWriter(file)
 	err = func() error {
-		// Create a versionEdit that gets us from an empty catalog to the current state.
+		// Create a VersionEdit that gets us from an empty catalog to the current state.
-		var ve versionEdit
+		var ve VersionEdit
 		ve.CreatorID = c.mu.creatorID
-		ve.NewObjects = make([]SharedObjectMetadata, 0, len(c.mu.objects))
+		ve.NewObjects = make([]RemoteObjectMetadata, 0, len(c.mu.objects))
 		for _, meta := range c.mu.objects {
 			ve.NewObjects = append(ve.NewObjects, meta)
 		}
@@ -361,7 +372,7 @@ func (c *Catalog) createNewCatalogFileLocked() (outErr error) {
 	return nil
 }
 
-func writeRecord(ve *versionEdit, file vfs.File, recWriter *record.Writer) error {
+func writeRecord(ve *VersionEdit, file vfs.File, recWriter *record.Writer) error {
 	w, err := recWriter.Next()
 	if err != nil {
 		return err

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/version_edit.go

@@ -2,7 +2,7 @@
 // of this source code is governed by a BSD-style license that can be found in
 // the LICENSE file.
 
-package sharedobjcat
+package remoteobjcat
 
 import (
 	"bufio"
@@ -11,28 +11,37 @@ import (
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/remote"
 )
 
-// versionEdit is a modification to the shared object state which can be encoded
+// VersionEdit is a modification to the remote object state which can be encoded
 // into a record.
 //
 // TODO(radu): consider adding creation and deletion time for debugging purposes.
-type versionEdit struct {
+type VersionEdit struct {
-	NewObjects     []SharedObjectMetadata
+	NewObjects     []RemoteObjectMetadata
 	DeletedObjects []base.DiskFileNum
 	CreatorID      objstorage.CreatorID
 }
 
 const (
-	// tagNewObject is followed by the FileNum, creator ID, creator FileNum, and
+	// tagNewObject is followed by the FileNum, creator ID, creator FileNum,
-	// cleanup method.
+	// cleanup method, optional new object tags, and ending with a 0 byte.
 	tagNewObject = 1
 	// tagDeletedObject is followed by the FileNum.
 	tagDeletedObject = 2
 	// tagCreatorID is followed by the Creator ID for this store. This ID can
 	// never change.
 	tagCreatorID = 3
+	// tagNewObjectLocator is an optional tag inside the tagNewObject payload. It
+	// is followed by the encoded length of the locator string and the string.
+	tagNewObjectLocator = 4
+	// tagNewObjectCustomName is an optional tag inside the tagNewObject payload.
+	// It is followed by the encoded length of the custom object name string
+	// followed by the string.
+	tagNewObjectCustomName = 5
 )
 
 // Object type values. We don't want to encode FileType directly because it is
@@ -61,8 +70,8 @@ func fileTypeToObjType(fileType base.FileType) (uint64, error) {
 }
 
 // Encode encodes an edit to the specified writer.
-func (v *versionEdit) Encode(w io.Writer) error {
+func (v *VersionEdit) Encode(w io.Writer) error {
-	buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*4+len(v.DeletedObjects)*2+2))
+	buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*10+len(v.DeletedObjects)*2+2))
 	for _, meta := range v.NewObjects {
 		objType, err := fileTypeToObjType(meta.FileType)
 		if err != nil {
@@ -74,6 +83,16 @@ func (v *versionEdit) Encode(w io.Writer) error {
 		buf = binary.AppendUvarint(buf, uint64(meta.CreatorID))
 		buf = binary.AppendUvarint(buf, uint64(meta.CreatorFileNum.FileNum()))
 		buf = binary.AppendUvarint(buf, uint64(meta.CleanupMethod))
+		if meta.Locator != "" {
+			buf = binary.AppendUvarint(buf, uint64(tagNewObjectLocator))
+			buf = encodeString(buf, string(meta.Locator))
+		}
+		if meta.CustomObjectName != "" {
+			buf = binary.AppendUvarint(buf, uint64(tagNewObjectCustomName))
+			buf = encodeString(buf, meta.CustomObjectName)
+		}
+		// Append 0 as the terminator for optional new object tags.
+		buf = binary.AppendUvarint(buf, 0)
 	}
 
 	for _, dfn := range v.DeletedObjects {
@@ -89,7 +108,7 @@ func (v *versionEdit) Encode(w io.Writer) error {
 }
 
 // Decode decodes an edit from the specified reader.
-func (v *versionEdit) Decode(r io.Reader) error {
+func (v *VersionEdit) Decode(r io.Reader) error {
 	br, ok := r.(io.ByteReader)
 	if !ok {
 		br = bufio.NewReader(r)
@@ -107,6 +126,7 @@ func (v *versionEdit) Decode(r io.Reader) error {
 		switch tag {
 		case tagNewObject:
 			var fileNum, creatorID, creatorFileNum, cleanupMethod uint64
+			var locator, customName string
 			var fileType base.FileType
 			fileNum, err = binary.ReadUvarint(br)
 			if err == nil {
@@ -125,13 +145,34 @@ func (v *versionEdit) Decode(r io.Reader) error {
 			if err == nil {
 				cleanupMethod, err = binary.ReadUvarint(br)
 			}
+			for err == nil {
+				var optionalTag uint64
+				optionalTag, err = binary.ReadUvarint(br)
+				if err != nil || optionalTag == 0 {
+					break
+				}
+
+				switch optionalTag {
+				case tagNewObjectLocator:
+					locator, err = decodeString(br)
+
+				case tagNewObjectCustomName:
+					customName, err = decodeString(br)
+
+				default:
+					err = errors.Newf("unknown newObject tag %d", optionalTag)
+				}
+			}
+
 			if err == nil {
-				v.NewObjects = append(v.NewObjects, SharedObjectMetadata{
+				v.NewObjects = append(v.NewObjects, RemoteObjectMetadata{
-					FileNum:        base.FileNum(fileNum).DiskFileNum(),
+					FileNum:          base.FileNum(fileNum).DiskFileNum(),
-					FileType:       fileType,
+					FileType:         fileType,
-					CreatorID:      objstorage.CreatorID(creatorID),
+					CreatorID:        objstorage.CreatorID(creatorID),
-					CreatorFileNum: base.FileNum(creatorFileNum).DiskFileNum(),
+					CreatorFileNum:   base.FileNum(creatorFileNum).DiskFileNum(),
-					CleanupMethod:  objstorage.SharedCleanupMethod(cleanupMethod),
+					CleanupMethod:    objstorage.SharedCleanupMethod(cleanupMethod),
+					Locator:          remote.Locator(locator),
+					CustomObjectName: customName,
 				})
 			}
 
@@ -150,8 +191,9 @@ func (v *versionEdit) Decode(r io.Reader) error {
 			}
 
 		default:
-			// Ignore unknown tags.
+			err = errors.Newf("unknown tag %d", tag)
 		}
+
 		if err != nil {
 			if err == io.EOF {
 				return errCorruptCatalog
@@ -162,4 +204,51 @@ func (v *versionEdit) Decode(r io.Reader) error {
 	return nil
 }
 
-var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt shared object catalog")
+func encodeString(buf []byte, s string) []byte {
+	buf = binary.AppendUvarint(buf, uint64(len(s)))
+	buf = append(buf, []byte(s)...)
+	return buf
+}
+
+func decodeString(br io.ByteReader) (string, error) {
+	length, err := binary.ReadUvarint(br)
+	if err != nil || length == 0 {
+		return "", err
+	}
+	buf := make([]byte, length)
+	for i := range buf {
+		buf[i], err = br.ReadByte()
+		if err != nil {
+			return "", err
+		}
+	}
+	return string(buf), nil
+}
+
+var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt remote object catalog")
+
+// Apply the version edit to a creator ID and a map of objects.
+func (v *VersionEdit) Apply(
+	creatorID *objstorage.CreatorID, objects map[base.DiskFileNum]RemoteObjectMetadata,
+) error {
+	if v.CreatorID.IsSet() {
+		*creatorID = v.CreatorID
+	}
+	for _, meta := range v.NewObjects {
+		if invariants.Enabled {
+			if _, exists := objects[meta.FileNum]; exists {
+				return errors.AssertionFailedf("version edit adds existing object %s", meta.FileNum)
+			}
+		}
+		objects[meta.FileNum] = meta
+	}
+	for _, fileNum := range v.DeletedObjects {
+		if invariants.Enabled {
+			if _, exists := objects[fileNum]; !exists {
+				return errors.AssertionFailedf("version edit deletes non-existent object %s", fileNum)
+			}
+		}
+		delete(objects, fileNum)
+	}
+	return nil
+}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared.go

@@ -1,284 +0,0 @@
-// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
-// of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package objstorageprovider
-
-import (
-	"context"
-	"runtime"
-	"sync"
-	"sync/atomic"
-
-	"github.com/cockroachdb/errors"
-	"github.com/cockroachdb/pebble/internal/base"
-	"github.com/cockroachdb/pebble/internal/invariants"
-	"github.com/cockroachdb/pebble/objstorage"
-	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
-	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat"
-	"github.com/cockroachdb/pebble/objstorage/shared"
-)
-
-// sharedSubsystem contains the provider fields related to shared storage.
-// All fields remain unset if shared storage is not configured.
-type sharedSubsystem struct {
-	catalog *sharedobjcat.Catalog
-	cache   *sharedcache.Cache
-
-	// checkRefsOnOpen controls whether we check the ref marker file when opening
-	// an object. Normally this is true when invariants are enabled (but the provider
-	// test tweaks this field).
-	checkRefsOnOpen bool
-
-	// initialized guards access to the creatorID field.
-	initialized atomic.Bool
-	creatorID   objstorage.CreatorID
-	initOnce    sync.Once
-}
-
-func (ss *sharedSubsystem) init(creatorID objstorage.CreatorID) {
-	ss.initOnce.Do(func() {
-		ss.creatorID = creatorID
-		ss.initialized.Store(true)
-	})
-}
-
-func (p *provider) sharedStorage() shared.Storage {
-	return p.st.Shared.Storage
-}
-
-// sharedInit initializes the shared object subsystem (if configured) and finds
-// any shared objects.
-func (p *provider) sharedInit() error {
-	if p.sharedStorage() == nil {
-		return nil
-	}
-	catalog, contents, err := sharedobjcat.Open(p.st.FS, p.st.FSDirName)
-	if err != nil {
-		return errors.Wrapf(err, "pebble: could not open shared object catalog")
-	}
-	p.shared.catalog = catalog
-	p.shared.checkRefsOnOpen = invariants.Enabled
-
-	// The creator ID may or may not be initialized yet.
-	if contents.CreatorID.IsSet() {
-		p.shared.init(contents.CreatorID)
-		p.st.Logger.Infof("shared storage configured; creatorID = %s", contents.CreatorID)
-	} else {
-		p.st.Logger.Infof("shared storage configured; no creatorID yet")
-	}
-
-	if p.st.Shared.CacheSizeBytes > 0 {
-		const defaultBlockSize = 32 * 1024
-		blockSize := p.st.Shared.CacheBlockSize
-		if blockSize == 0 {
-			blockSize = defaultBlockSize
-		}
-
-		numShards := p.st.Shared.CacheShardCount
-		if numShards == 0 {
-			numShards = 2 * runtime.GOMAXPROCS(0)
-		}
-
-		p.shared.cache, err = sharedcache.Open(p.st.FS, p.st.Logger, p.st.FSDirName, blockSize, p.st.Shared.CacheSizeBytes, numShards)
-		if err != nil {
-			return errors.Wrapf(err, "pebble: could not open shared object cache")
-		}
-	}
-
-	for _, meta := range contents.Objects {
-		o := objstorage.ObjectMetadata{
-			DiskFileNum: meta.FileNum,
-			FileType:    meta.FileType,
-		}
-		o.Shared.CreatorID = meta.CreatorID
-		o.Shared.CreatorFileNum = meta.CreatorFileNum
-		o.Shared.CleanupMethod = meta.CleanupMethod
-		p.mu.knownObjects[o.DiskFileNum] = o
-	}
-	return nil
-}
-
-// SetCreatorID is part of the objstorage.Provider interface.
-func (p *provider) SetCreatorID(creatorID objstorage.CreatorID) error {
-	if p.sharedStorage() == nil {
-		return errors.AssertionFailedf("attempt to set CreatorID but shared storage not enabled")
-	}
-	// Note: this call is a cheap no-op if the creator ID was already set. This
-	// call also checks if we are trying to change the ID.
-	if err := p.shared.catalog.SetCreatorID(creatorID); err != nil {
-		return err
-	}
-	if !p.shared.initialized.Load() {
-		p.st.Logger.Infof("shared storage creatorID set to %s", creatorID)
-		p.shared.init(creatorID)
-	}
-	return nil
-}
-
-// IsForeign is part of the objstorage.Provider interface.
-func (p *provider) IsForeign(meta objstorage.ObjectMetadata) bool {
-	if !p.shared.initialized.Load() {
-		return false
-	}
-	return meta.IsShared() && p.shared.creatorID != meta.Shared.CreatorID
-}
-
-func (p *provider) sharedCheckInitialized() error {
-	if p.sharedStorage() == nil {
-		return errors.Errorf("shared object support not configured")
-	}
-	if !p.shared.initialized.Load() {
-		return errors.Errorf("shared object support not available: shared creator ID not yet set")
-	}
-	return nil
-}
-
-func (p *provider) sharedSync() error {
-	batch := func() sharedobjcat.Batch {
-		p.mu.Lock()
-		defer p.mu.Unlock()
-		res := p.mu.shared.catalogBatch.Copy()
-		p.mu.shared.catalogBatch.Reset()
-		return res
-	}()
-
-	if batch.IsEmpty() {
-		return nil
-	}
-
-	err := p.shared.catalog.ApplyBatch(batch)
-	if err != nil {
-		// We have to put back the batch (for the next Sync).
-		p.mu.Lock()
-		defer p.mu.Unlock()
-		batch.Append(p.mu.shared.catalogBatch)
-		p.mu.shared.catalogBatch = batch
-		return err
-	}
-
-	return nil
-}
-
-func (p *provider) sharedPath(meta objstorage.ObjectMetadata) string {
-	return "shared://" + sharedObjectName(meta)
-}
-
-// sharedCreateRef creates a reference marker object.
-func (p *provider) sharedCreateRef(meta objstorage.ObjectMetadata) error {
-	if meta.Shared.CleanupMethod != objstorage.SharedRefTracking {
-		return nil
-	}
-	refName := p.sharedObjectRefName(meta)
-	writer, err := p.sharedStorage().CreateObject(refName)
-	if err == nil {
-		// The object is empty, just close the writer.
-		err = writer.Close()
-	}
-	if err != nil {
-		return errors.Wrapf(err, "creating marker object %q", refName)
-	}
-	return nil
-}
-
-func (p *provider) sharedCreate(
-	_ context.Context,
-	fileType base.FileType,
-	fileNum base.DiskFileNum,
-	opts objstorage.CreateOptions,
-) (objstorage.Writable, objstorage.ObjectMetadata, error) {
-	if err := p.sharedCheckInitialized(); err != nil {
-		return nil, objstorage.ObjectMetadata{}, err
-	}
-	meta := objstorage.ObjectMetadata{
-		DiskFileNum: fileNum,
-		FileType:    fileType,
-	}
-	meta.Shared.CreatorID = p.shared.creatorID
-	meta.Shared.CreatorFileNum = fileNum
-	meta.Shared.CleanupMethod = opts.SharedCleanupMethod
-
-	objName := sharedObjectName(meta)
-	writer, err := p.sharedStorage().CreateObject(objName)
-	if err != nil {
-		return nil, objstorage.ObjectMetadata{}, errors.Wrapf(err, "creating object %q", objName)
-	}
-	return &sharedWritable{
-		p:             p,
-		meta:          meta,
-		storageWriter: writer,
-	}, meta, nil
-}
-
-func (p *provider) sharedOpenForReading(
-	ctx context.Context, meta objstorage.ObjectMetadata, opts objstorage.OpenOptions,
-) (objstorage.Readable, error) {
-	if err := p.sharedCheckInitialized(); err != nil {
-		return nil, err
-	}
-	// Verify we have a reference on this object; for performance reasons, we only
-	// do this in testing scenarios.
-	if p.shared.checkRefsOnOpen && meta.Shared.CleanupMethod == objstorage.SharedRefTracking {
-		refName := p.sharedObjectRefName(meta)
-		if _, err := p.sharedStorage().Size(refName); err != nil {
-			if p.sharedStorage().IsNotExistError(err) {
-				if opts.MustExist {
-					p.st.Logger.Fatalf("marker object %q does not exist", refName)
-					// TODO(radu): maybe list references for the object.
-				}
-				return nil, errors.Errorf("marker object %q does not exist", refName)
-			}
-			return nil, errors.Wrapf(err, "checking marker object %q", refName)
-		}
-	}
-	objName := sharedObjectName(meta)
-	reader, size, err := p.sharedStorage().ReadObject(ctx, objName)
-	if err != nil {
-		if opts.MustExist && p.sharedStorage().IsNotExistError(err) {
-			p.st.Logger.Fatalf("object %q does not exist", objName)
-			// TODO(radu): maybe list references for the object.
-		}
-		return nil, err
-	}
-	return p.newSharedReadable(reader, size, meta.DiskFileNum), nil
-}
-
-func (p *provider) sharedSize(meta objstorage.ObjectMetadata) (int64, error) {
-	if err := p.sharedCheckInitialized(); err != nil {
-		return 0, err
-	}
-	objName := sharedObjectName(meta)
-	return p.sharedStorage().Size(objName)
-}
-
-// sharedUnref implements object "removal" with the shared backend. The ref
-// marker object is removed and the backing object is removed only if there are
-// no other ref markers.
-func (p *provider) sharedUnref(meta objstorage.ObjectMetadata) error {
-	if meta.Shared.CleanupMethod == objstorage.SharedNoCleanup {
-		// Never delete objects in this mode.
-		return nil
-	}
-	if p.isProtected(meta.DiskFileNum) {
-		// TODO(radu): we need a mechanism to unref the object when it becomes
-		// unprotected.
-		return nil
-	}
-
-	refName := p.sharedObjectRefName(meta)
-	// Tolerate a not-exists error.
-	if err := p.sharedStorage().Delete(refName); err != nil && !p.sharedStorage().IsNotExistError(err) {
-		return err
-	}
-	otherRefs, err := p.sharedStorage().List(sharedObjectRefPrefix(meta), "" /* delimiter */)
-	if err != nil {
-		return err
-	}
-	if len(otherRefs) == 0 {
-		objName := sharedObjectName(meta)
-		if err := p.sharedStorage().Delete(objName); err != nil && !p.sharedStorage().IsNotExistError(err) {
-			return err
-		}
-	}
-	return nil
-}