Update deps

go.mod

@@ -4,40 +4,38 @@ go 1.21
 
 require (
 	github.com/cockroachdb/errors v1.11.1
-	github.com/cockroachdb/pebble v0.0.0-20230905214401-0cbd3b1bccef
+	github.com/cockroachdb/pebble v1.1.0
 	github.com/davecgh/go-spew v1.1.1
-	github.com/eleksir/go-xmpp v1.5.0
+	github.com/eleksir/go-xmpp v1.6.0
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/hjson/hjson-go v3.3.0+incompatible
 	github.com/sirupsen/logrus v1.9.3
-	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
+	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
 )
 
 require (
 	github.com/DataDog/zstd v1.5.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
 	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/getsentry/sentry-go v0.24.0 // indirect
+	github.com/getsentry/sentry-go v0.27.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/client_golang v1.19.0 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/common v0.53.0 // indirect
-	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/prometheus/procfs v0.14.0 // indirect
-	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
-	golang.org/x/crypto v0.13.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 )

go.sum

@@ -2,16 +2,16 @@ github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
 github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
 github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
 github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8=
 github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20230905214401-0cbd3b1bccef h1:woeEFsEoclqyaKA0uNlotcy+pMDrULq5FZptvmb9Dik=
+github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4=
-github.com/cockroachdb/pebble v0.0.0-20230905214401-0cbd3b1bccef/go.mod h1:nindLFinxeDPjP4qI9LtVHAwDef57/0s5KMfWgdktQc=
+github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E=
 github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
@@ -22,39 +22,32 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/eleksir/go-xmpp v1.5.0 h1:kAjpCxuqWiu9LdlCTkQzvczotHONpojwp/mpI4aBVmQ=
+github.com/eleksir/go-xmpp v1.6.0 h1:bkGr0YVczJWpkyk/IHhZb2ewv+NeHLvdNfds0TFgqao=
-github.com/eleksir/go-xmpp v1.5.0/go.mod h1:KnvbXvUpttEVFl8l5eDO2Q7J97ZxtTMvOsDWULD2Rn8=
+github.com/eleksir/go-xmpp v1.6.0/go.mod h1:Ii6Wemk+yuGQOF+Glt0syjvtcPvdaBLstuNp/+rRJrg=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/getsentry/sentry-go v0.24.0 h1:02b7qEmJ56EHGe9KFgjArjU/vG/aywm7Efgu+iPc01Y=
+github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
-github.com/getsentry/sentry-go v0.24.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
+github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hjson/hjson-go v3.3.0+incompatible h1:Rqr+Ya+0aCJMjaE4s8E9YKvuJLuLVpEvz4ONum52vnI=
 github.com/hjson/hjson-go v3.3.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
-github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
-github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
@@ -68,17 +61,17 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
-github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
-github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
-github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
-github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE=
-github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U=
-github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
+github.com/prometheus/procfs v0.14.0 h1:Lw4VdGGoKEZilJsayHf0B+9YgLGREba2C6xr+Fdfq6s=
-github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
+github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy5Mu67pHlNQ=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
-github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
-github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -90,34 +83,33 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
-golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
+golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
-golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
+golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
@@ -126,10 +118,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

vendor/github.com/cespare/xxhash/v2/README.md

@@ -70,3 +70,5 @@ benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$')
 - [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
 - [FreeCache](https://github.com/coocood/freecache)
 - [FastCache](https://github.com/VictoriaMetrics/fastcache)
+- [Ristretto](https://github.com/dgraph-io/ristretto)
+- [Badger](https://github.com/dgraph-io/badger)

vendor/github.com/cespare/xxhash/v2/xxhash.go

@@ -19,10 +19,13 @@ const (
 // Store the primes in an array as well.
 //
 // The consts are used when possible in Go code to avoid MOVs but we need a
-// contiguous array of the assembly code.
+// contiguous array for the assembly code.
 var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5}
 
 // Digest implements hash.Hash64.
+//
+// Note that a zero-valued Digest is not ready to receive writes.
+// Call Reset or create a Digest using New before calling other methods.
 type Digest struct {
 	v1    uint64
 	v2    uint64
@@ -33,19 +36,31 @@ type Digest struct {
 	n     int // how much of mem is used
 }
 
-// New creates a new Digest that computes the 64-bit xxHash algorithm.
+// New creates a new Digest with a zero seed.
 func New() *Digest {
+	return NewWithSeed(0)
+}
+
+// NewWithSeed creates a new Digest with the given seed.
+func NewWithSeed(seed uint64) *Digest {
 	var d Digest
-	d.Reset()
+	d.ResetWithSeed(seed)
 	return &d
 }
 
 // Reset clears the Digest's state so that it can be reused.
+// It uses a seed value of zero.
 func (d *Digest) Reset() {
-	d.v1 = primes[0] + prime2
+	d.ResetWithSeed(0)
-	d.v2 = prime2
+}
-	d.v3 = 0
+
-	d.v4 = -primes[0]
+// ResetWithSeed clears the Digest's state so that it can be reused.
+// It uses the given seed to initialize the state.
+func (d *Digest) ResetWithSeed(seed uint64) {
+	d.v1 = seed + prime1 + prime2
+	d.v2 = seed + prime2
+	d.v3 = seed
+	d.v4 = seed - prime1
 	d.total = 0
 	d.n = 0
 }

vendor/github.com/cespare/xxhash/v2/xxhash_asm.go

@@ -6,7 +6,7 @@
 
 package xxhash
 
-// Sum64 computes the 64-bit xxHash digest of b.
+// Sum64 computes the 64-bit xxHash digest of b with a zero seed.
 //
 //go:noescape
 func Sum64(b []byte) uint64

vendor/github.com/cespare/xxhash/v2/xxhash_other.go

@@ -3,7 +3,7 @@
 
 package xxhash
 
-// Sum64 computes the 64-bit xxHash digest of b.
+// Sum64 computes the 64-bit xxHash digest of b with a zero seed.
 func Sum64(b []byte) uint64 {
 	// A simpler version would be
 	//   d := New()

vendor/github.com/cespare/xxhash/v2/xxhash_safe.go

@@ -5,7 +5,7 @@
 
 package xxhash
 
-// Sum64String computes the 64-bit xxHash digest of s.
+// Sum64String computes the 64-bit xxHash digest of s with a zero seed.
 func Sum64String(s string) uint64 {
 	return Sum64([]byte(s))
 }

vendor/github.com/cespare/xxhash/v2/xxhash_unsafe.go

@@ -33,7 +33,7 @@ import (
 //
 // See https://github.com/golang/go/issues/42739 for discussion.
 
-// Sum64String computes the 64-bit xxHash digest of s.
+// Sum64String computes the 64-bit xxHash digest of s with a zero seed.
 // It may be faster than Sum64([]byte(s)) by avoiding a copy.
 func Sum64String(s string) uint64 {
 	b := *(*[]byte)(unsafe.Pointer(&sliceHeader{s, len(s)}))

vendor/github.com/cockroachdb/pebble/Makefile

@@ -4,7 +4,7 @@ GOFLAGS :=
 STRESSFLAGS :=
 TAGS := invariants
 TESTS := .
-LATEST_RELEASE := $(shell git fetch origin && git branch -r --list '*/crl-release-*' | grep -o 'crl-release-.*$$' | sort | tail -1)
+PREV_RELEASE := crl-release-23.1
 COVER_PROFILE := coverprofile.out
 
 .PHONY: all
@@ -63,11 +63,11 @@ stressmeta: stress
 
 .PHONY: crossversion-meta
 crossversion-meta:
-	git checkout ${LATEST_RELEASE}; \
+	git checkout ${PREV_RELEASE}; \
-		${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${LATEST_RELEASE}.test'; \
+		${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${PREV_RELEASE}.test'; \
 		git checkout -; \
 		${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/head.test'; \
-		${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${LATEST_RELEASE},${LATEST_RELEASE},${LATEST_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
+		${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${PREV_RELEASE},${PREV_RELEASE},${PREV_RELEASE}.test' --version 'HEAD,HEAD,./head.test'
 
 .PHONY: stress-crossversion
 stress-crossversion:

vendor/github.com/cockroachdb/pebble/README.md

@@ -1,4 +1,4 @@
-# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble)
+# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble) <sup><sub><sub>[Coverage](https://storage.googleapis.com/crl-codecover-public/pebble/index.html)</sub></sub></sup>
 
 #### [Nightly benchmarks](https://cockroachdb.github.io/pebble/)
 
@@ -8,9 +8,8 @@ RocksDB file formats and a few extensions such as range deletion
 tombstones, table-level bloom filters, and updates to the MANIFEST
 format.
 
-Pebble intentionally does not aspire to include every feature in
+Pebble intentionally does not aspire to include every feature in RocksDB and
-RocksDB and is specifically targetting the use case and feature set
+specifically targets the use case and feature set needed by CockroachDB:
-needed by CockroachDB:
 
 * Block-based tables
 * Checkpoints
@@ -68,6 +67,17 @@ Pebble offers several improvements over RocksDB:
 * Faster commit pipeline that achieves better concurrency.
 * Seamless merged iteration of indexed batches. The mutations in the
   batch conceptually occupy another memtable level.
+* L0 sublevels and flush splitting for concurrent compactions out of L0 and
+  reduced read-amplification during heavy write load.
+* Faster LSM edits in LSMs with large numbers of sstables through use of a
+  copy-on-write B-tree to hold file metadata.
+* Delete-only compactions that drop whole sstables that fall within the bounds
+  of a range deletion.
+* Block-property collectors and filters that enable iterators to skip tables,
+  index blocks and data blocks that are irrelevant, according to user-defined
+  properties over key-value pairs.
+* Range keys API, allowing KV pairs defined over a range of keyspace with
+  user-defined semantics and interleaved during iteration.
 * Smaller, more approachable code base.
 
 See the [Pebble vs RocksDB: Implementation
@@ -100,10 +110,68 @@ known incompatibilities.
   format.
 * Plain table format. Pebble does not support the plain table sstable
   format.
-* SSTable format version 3 and 4. Pebble does not currently support
+* SSTable format version 3 and 4. Pebble does not support version 3
-  version 3 and version 4 format sstables. The sstable format version
+  and version 4 format sstables. The sstable format version is
-  is controlled by the `BlockBasedTableOptions::format_version`
+  controlled by the `BlockBasedTableOptions::format_version` option.
-  option. See [#97](https://github.com/cockroachdb/pebble/issues/97).
+  See [#97](https://github.com/cockroachdb/pebble/issues/97).
+
+## Format major versions
+
+Over time Pebble has introduced new physical file formats.  Backwards
+incompatible changes are made through the introduction of 'format major
+versions'. By default, when Pebble opens a database, it defaults to
+`FormatMostCompatible`. This version is bi-directionally compatible with RocksDB
+6.2.1 (with the caveats described above).
+
+To opt into new formats, a user may set `FormatMajorVersion` on the
+[`Options`](https://pkg.go.dev/github.com/cockroachdb/pebble#Options)
+supplied to
+[`Open`](https://pkg.go.dev/github.com/cockroachdb/pebble#Open), or
+upgrade the format major version at runtime using
+[`DB.RatchetFormatMajorVersion`](https://pkg.go.dev/github.com/cockroachdb/pebble#DB.RatchetFormatMajorVersion).
+Format major version upgrades are permanent; There is no option to
+return to an earlier format.
+
+The table below outlines the history of format major versions:
+
+| Name                               | Value | Migration  |
+|------------------------------------|-------|------------|
+| FormatMostCompatible               |   1   | No         |
+| FormatVersioned                    |   3   | No         |
+| FormatSetWithDelete                |   4   | No         |
+| FormatBlockPropertyCollector       |   5   | No         |
+| FormatSplitUserKeysMarked          |   6   | Background |
+| FormatSplitUserKeysMarkedCompacted |   7   | Blocking   |
+| FormatRangeKeys                    |   8   | No         |
+| FormatMinTableFormatPebblev1       |   9   | No         |
+| FormatPrePebblev1Marked            |  10   | Background |
+| FormatSSTableValueBlocks           |  12   | No         |
+| FormatFlushableIngest              |  13   | No         |
+| FormatPrePebblev1MarkedCompacted   |  14   | Blocking   |
+| FormatDeleteSizedAndObsolete       |  15   | No         |
+| FormatVirtualSSTables              |  16   | No         |
+
+Upgrading to a format major version with 'Background' in the migration
+column may trigger background activity to rewrite physical file
+formats, typically through compactions. Upgrading to a format major
+version with 'Blocking' in the migration column will block until a
+migration is complete. The database may continue to serve reads and
+writes if upgrading a live database through
+`RatchetFormatMajorVersion`, but the method call will not return until
+the migration is complete.
+
+For reference, the table below lists the range of supported Pebble format major
+versions for CockroachDB releases.
+
+| CockroachDB release | Earliest supported                 | Latest supported          |
+|---------------------|------------------------------------|---------------------------|
+| 20.1 through 21.1   | FormatMostCompatible               | FormatMostCompatible      |
+| 21.2                | FormatMostCompatible               | FormatSetWithDelete       |
+| 21.2                | FormatMostCompatible               | FormatSetWithDelete       |
+| 22.1                | FormatMostCompatible               | FormatSplitUserKeysMarked |
+| 22.2                | FormatMostCompatible               | FormatPrePebblev1Marked   |
+| 23.1                | FormatSplitUserKeysMarkedCompacted | FormatFlushableIngest     |
+| 23.2                | FormatSplitUserKeysMarkedCompacted | FormatVirtualSSTables     |
 
 ## Pedigree
 

vendor/github.com/cockroachdb/pebble/batch.go

@@ -41,10 +41,10 @@ const (
 var ErrNotIndexed = errors.New("pebble: batch not indexed")
 
 // ErrInvalidBatch indicates that a batch is invalid or otherwise corrupted.
-var ErrInvalidBatch = errors.New("pebble: invalid batch")
+var ErrInvalidBatch = base.MarkCorruptionError(errors.New("pebble: invalid batch"))
 
 // ErrBatchTooLarge indicates that a batch is invalid or otherwise corrupted.
-var ErrBatchTooLarge = errors.Newf("pebble: batch too large: >= %s", humanize.Bytes.Uint64(maxBatchSize))
+var ErrBatchTooLarge = base.MarkCorruptionError(errors.Newf("pebble: batch too large: >= %s", humanize.Bytes.Uint64(maxBatchSize)))
 
 // DeferredBatchOp represents a batch operation (eg. set, merge, delete) that is
 // being inserted into the batch. Indexing is not performed on the specified key
@@ -183,6 +183,13 @@ func (d DeferredBatchOp) Finish() error {
 // WAL, and thus stable. New record kinds may be added, but the existing ones
 // will not be modified.
 type Batch struct {
+	batchInternal
+	applied atomic.Bool
+}
+
+// batchInternal contains the set of fields within Batch that are non-atomic and
+// capable of being reset using a *b = batchInternal{} struct copy.
+type batchInternal struct {
 	// Data is the wire format of a batch's log entry:
 	//   - 8 bytes for a sequence number of the first batch element,
 	//     or zeroes if the batch has not yet been applied,
@@ -260,11 +267,6 @@ type Batch struct {
 	// memtable.
 	flushable *flushableBatch
 
-	// ingestedSSTBatch indicates that the batch contains one or more key kinds
-	// of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST
-	// then it will only contain key kinds of IngestSST.
-	ingestedSSTBatch bool
-
 	// minimumFormatMajorVersion indicates the format major version required in
 	// order to commit this batch. If an operation requires a particular format
 	// major version, it ratchets the batch's minimumFormatMajorVersion. When
@@ -292,7 +294,22 @@ type Batch struct {
 	commitStats BatchCommitStats
 
 	commitErr error
-	applied   atomic.Bool
+
+	// Position bools together to reduce the sizeof the struct.
+
+	// ingestedSSTBatch indicates that the batch contains one or more key kinds
+	// of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST
+	// then it will only contain key kinds of IngestSST.
+	ingestedSSTBatch bool
+
+	// committing is set to true when a batch begins to commit. It's used to
+	// ensure the batch is not mutated concurrently. It is not an atomic
+	// deliberately, so as to avoid the overhead on batch mutations. This is
+	// okay, because under correct usage this field will never be accessed
+	// concurrently. It's only under incorrect usage the memory accesses of this
+	// variable may violate memory safety. Since we don't use atomics here,
+	// false negatives are possible.
+	committing bool
 }
 
 // BatchCommitStats exposes stats related to committing a batch.
@@ -432,18 +449,21 @@ func (b *Batch) release() {
 	}
 }
 
-func (b *Batch) refreshMemTableSize() {
+func (b *Batch) refreshMemTableSize() error {
 	b.memTableSize = 0
 	if len(b.data) < batchHeaderLen {
-		return
+		return nil
 	}
 
 	b.countRangeDels = 0
 	b.countRangeKeys = 0
 	b.minimumFormatMajorVersion = 0
 	for r := b.Reader(); ; {
-		kind, key, value, ok := r.Next()
+		kind, key, value, ok, err := r.Next()
 		if !ok {
+			if err != nil {
+				return err
+			}
 			break
 		}
 		switch kind {
@@ -452,8 +472,8 @@ func (b *Batch) refreshMemTableSize() {
 		case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
 			b.countRangeKeys++
 		case InternalKeyKindDeleteSized:
-			if b.minimumFormatMajorVersion < ExperimentalFormatDeleteSizedAndObsolete {
+			if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete {
-				b.minimumFormatMajorVersion = ExperimentalFormatDeleteSizedAndObsolete
+				b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete
 			}
 		case InternalKeyKindIngestSST:
 			if b.minimumFormatMajorVersion < FormatFlushableIngest {
@@ -467,6 +487,7 @@ func (b *Batch) refreshMemTableSize() {
 	if b.countRangeKeys > 0 && b.minimumFormatMajorVersion < FormatRangeKeys {
 		b.minimumFormatMajorVersion = FormatRangeKeys
 	}
+	return nil
 }
 
 // Apply the operations contained in the batch to the receiver batch.
@@ -480,7 +501,7 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error {
 		return nil
 	}
 	if len(batch.data) < batchHeaderLen {
-		return base.CorruptionErrorf("pebble: invalid batch")
+		return ErrInvalidBatch
 	}
 
 	offset := len(b.data)
@@ -497,8 +518,11 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error {
 		// order to update the index.
 		for iter := BatchReader(b.data[offset:]); len(iter) > 0; {
 			offset := uintptr(unsafe.Pointer(&iter[0])) - uintptr(unsafe.Pointer(&b.data[0]))
-			kind, key, value, ok := iter.Next()
+			kind, key, value, ok, err := iter.Next()
 			if !ok {
+				if err != nil {
+					return err
+				}
 				break
 			}
 			switch kind {
@@ -554,6 +578,9 @@ func (b *Batch) Get(key []byte) ([]byte, io.Closer, error) {
 }
 
 func (b *Batch) prepareDeferredKeyValueRecord(keyLen, valueLen int, kind InternalKeyKind) {
+	if b.committing {
+		panic("pebble: batch already committing")
+	}
 	if len(b.data) == 0 {
 		b.init(keyLen + valueLen + 2*binary.MaxVarintLen64 + batchHeaderLen)
 	}
@@ -603,6 +630,9 @@ func (b *Batch) prepareDeferredKeyValueRecord(keyLen, valueLen int, kind Interna
 }
 
 func (b *Batch) prepareDeferredKeyRecord(keyLen int, kind InternalKeyKind) {
+	if b.committing {
+		panic("pebble: batch already committing")
+	}
 	if len(b.data) == 0 {
 		b.init(keyLen + binary.MaxVarintLen64 + batchHeaderLen)
 	}
@@ -784,8 +814,8 @@ func (b *Batch) DeleteSized(key []byte, deletedValueSize uint32, _ *WriteOptions
 // complete key slice, letting the caller encode into the DeferredBatchOp.Key
 // slice and then call Finish() on the returned object.
 func (b *Batch) DeleteSizedDeferred(keyLen int, deletedValueSize uint32) *DeferredBatchOp {
-	if b.minimumFormatMajorVersion < ExperimentalFormatDeleteSizedAndObsolete {
+	if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete {
-		b.minimumFormatMajorVersion = ExperimentalFormatDeleteSizedAndObsolete
+		b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete
 	}
 
 	// Encode the sum of the key length and the value in the value.
@@ -1073,11 +1103,12 @@ func (b *Batch) SetRepr(data []byte) error {
 	}
 	b.data = data
 	b.count = uint64(binary.LittleEndian.Uint32(b.countData()))
+	var err error
 	if b.db != nil {
 		// Only track memTableSize for batches that will be committed to the DB.
-		b.refreshMemTableSize()
+		err = b.refreshMemTableSize()
 	}
-	return nil
+	return err
 }
 
 // NewIter returns an iterator that is unpositioned (Iterator.Valid() will
@@ -1320,33 +1351,30 @@ func (b *Batch) init(size int) {
 	if cap(b.data) < n {
 		b.data = rawalloc.New(batchHeaderLen, n)
 	}
-	b.setCount(0)
-	b.setSeqNum(0)
 	b.data = b.data[:batchHeaderLen]
+	// Zero the sequence number in the header.
+	for i := 0; i < len(b.data); i++ {
+		b.data[i] = 0
+	}
 }
 
 // Reset resets the batch for reuse. The underlying byte slice (that is
-// returned by Repr()) is not modified. It is only necessary to call this
+// returned by Repr()) may not be modified. It is only necessary to call this
 // method if a batch is explicitly being reused. Close automatically takes are
 // of releasing resources when appropriate for batches that are internally
 // being reused.
 func (b *Batch) Reset() {
-	b.count = 0
+	// Zero out the struct, retaining only the fields necessary for manual
-	b.countRangeDels = 0
+	// reuse.
-	b.countRangeKeys = 0
+	b.batchInternal = batchInternal{
-	b.memTableSize = 0
+		data:           b.data,
-	b.deferredOp = DeferredBatchOp{}
+		cmp:            b.cmp,
-	b.tombstones = nil
+		formatKey:      b.formatKey,
-	b.tombstonesSeqNum = 0
+		abbreviatedKey: b.abbreviatedKey,
-	b.rangeKeys = nil
+		index:          b.index,
-	b.rangeKeysSeqNum = 0
+		db:             b.db,
-	b.flushable = nil
+	}
-	b.commit = sync.WaitGroup{}
-	b.fsyncWait = sync.WaitGroup{}
-	b.commitStats = BatchCommitStats{}
-	b.commitErr = nil
 	b.applied.Store(false)
-	b.minimumFormatMajorVersion = 0
 	if b.data != nil {
 		if cap(b.data) > batchMaxRetainedSize {
 			// If the capacity of the buffer is larger than our maximum
@@ -1357,13 +1385,14 @@ func (b *Batch) Reset() {
 		} else {
 			// Otherwise, reset the buffer for re-use.
 			b.data = b.data[:batchHeaderLen]
-			b.setSeqNum(0)
+			// Zero the sequence number in the header.
+			for i := 0; i < len(b.data); i++ {
+				b.data[i] = 0
+			}
 		}
 	}
 	if b.index != nil {
 		b.index.Init(&b.data, b.cmp, b.abbreviatedKey)
-		b.rangeDelIndex = nil
-		b.rangeKeyIndex = nil
 	}
 }
 
@@ -1435,6 +1464,11 @@ func (b *Batch) Reader() BatchReader {
 }
 
 func batchDecodeStr(data []byte) (odata []byte, s []byte, ok bool) {
+	// TODO(jackson): This will index out of bounds if there's no varint or an
+	// invalid varint (eg, a single 0xff byte). Correcting will add a bit of
+	// overhead. We could avoid that overhead whenever len(data) >=
+	// binary.MaxVarint32?
+
 	var v uint32
 	var n int
 	ptr := unsafe.Pointer(&data[0])
@@ -1497,19 +1531,21 @@ func ReadBatch(repr []byte) (r BatchReader, count uint32) {
 	return repr[batchHeaderLen:], count
 }
 
-// Next returns the next entry in this batch. The final return value is false
+// Next returns the next entry in this batch, if there is one. If the reader has
-// if the batch is corrupt. The end of batch is reached when len(r)==0.
+// reached the end of the batch, Next returns ok=false and a nil error. If the
-func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, ok bool) {
+// batch is corrupt and the next entry is illegible, Next returns ok=false and a
+// non-nil error.
+func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, ok bool, err error) {
 	if len(*r) == 0 {
-		return 0, nil, nil, false
+		return 0, nil, nil, false, nil
 	}
 	kind = InternalKeyKind((*r)[0])
 	if kind > InternalKeyKindMax {
-		return 0, nil, nil, false
+		return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "invalid key kind 0x%x", (*r)[0])
 	}
 	*r, ukey, ok = batchDecodeStr((*r)[1:])
 	if !ok {
-		return 0, nil, nil, false
+		return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "decoding user key")
 	}
 	switch kind {
 	case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete,
@@ -1517,10 +1553,10 @@ func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, o
 		InternalKeyKindDeleteSized:
 		*r, value, ok = batchDecodeStr(*r)
 		if !ok {
-			return 0, nil, nil, false
+			return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "decoding %s value", kind)
 		}
 	}
-	return kind, ukey, value, true
+	return kind, ukey, value, true, nil
 }
 
 // Note: batchIter mirrors the implementation of flushableBatchIter. Keep the
@@ -1722,7 +1758,7 @@ var _ flushable = (*flushableBatch)(nil)
 // interface. This allows the batch to act like a memtable and be placed in the
 // queue of flushable memtables. Note that the flushable batch takes ownership
 // of the batch data.
-func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
+func newFlushableBatch(batch *Batch, comparer *Comparer) (*flushableBatch, error) {
 	b := &flushableBatch{
 		data:      batch.data,
 		cmp:       comparer.Compare,
@@ -1743,8 +1779,11 @@ func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
 		var index uint32
 		for iter := BatchReader(b.data[batchHeaderLen:]); len(iter) > 0; index++ {
 			offset := uintptr(unsafe.Pointer(&iter[0])) - uintptr(unsafe.Pointer(&b.data[0]))
-			kind, key, _, ok := iter.Next()
+			kind, key, _, ok, err := iter.Next()
 			if !ok {
+				if err != nil {
+					return nil, err
+				}
 				break
 			}
 			entry := flushableBatchEntry{
@@ -1816,7 +1855,7 @@ func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch {
 		}
 		fragmentRangeKeys(frag, it, len(rangeKeyOffsets))
 	}
-	return b
+	return b, nil
 }
 
 func (b *flushableBatch) setSeqNum(seqNum uint64) {
@@ -2242,7 +2281,10 @@ func batchSort(
 		rangeKeyIter := b.newRangeKeyIter(nil, math.MaxUint64)
 		return pointIter, rangeDelIter, rangeKeyIter
 	}
-	f := newFlushableBatch(b, b.db.opts.Comparer)
+	f, err := newFlushableBatch(b, b.db.opts.Comparer)
+	if err != nil {
+		panic(err)
+	}
 	return f.newIter(nil), f.newRangeDelIter(nil), f.newRangeKeyIter(nil)
 }
 

vendor/github.com/cockroachdb/pebble/checkpoint.go

@@ -135,9 +135,6 @@ func mkdirAllAndSyncParents(fs vfs.FS, destDir string) (vfs.File, error) {
 // space overhead for a checkpoint if hard links are disabled. Also beware that
 // even if hard links are used, the space overhead for the checkpoint will
 // increase over time as the DB performs compactions.
-//
-// TODO(bananabrick): Test checkpointing of virtual sstables once virtual
-// sstables is running e2e.
 func (d *DB) Checkpoint(
 	destDir string, opts ...CheckpointOption,
 ) (
@@ -192,7 +189,7 @@ func (d *DB) Checkpoint(
 	manifestSize := d.mu.versions.manifest.Size()
 	optionsFileNum := d.optionsFileNum
 	virtualBackingFiles := make(map[base.DiskFileNum]struct{})
-	for diskFileNum := range d.mu.versions.fileBackingMap {
+	for diskFileNum := range d.mu.versions.backingState.fileBackingMap {
 		virtualBackingFiles[diskFileNum] = struct{}{}
 	}
 	// Release the manifest and DB.mu so we don't block other operations on

vendor/github.com/cockroachdb/pebble/compaction_iter.go

@@ -15,9 +15,9 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/bytealloc"
-	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/rangekey"
+	"github.com/cockroachdb/redact"
 )
 
 // compactionIter provides a forward-only iterator that encapsulates the logic
@@ -256,10 +256,12 @@ type compactionIter struct {
 	// The fragmented range keys.
 	rangeKeys []keyspan.Span
 	// Byte allocator for the tombstone keys.
-	alloc               bytealloc.A
+	alloc                                  bytealloc.A
-	allowZeroSeqNum     bool
+	allowZeroSeqNum                        bool
-	elideTombstone      func(key []byte) bool
+	elideTombstone                         func(key []byte) bool
-	elideRangeTombstone func(start, end []byte) bool
+	elideRangeTombstone                    func(start, end []byte) bool
+	ineffectualSingleDeleteCallback        func(userKey []byte)
+	singleDeleteInvariantViolationCallback func(userKey []byte)
 	// The on-disk format major version. This informs the types of keys that
 	// may be written to disk during a compaction.
 	formatVersion FormatMajorVersion
@@ -281,20 +283,24 @@ func newCompactionIter(
 	allowZeroSeqNum bool,
 	elideTombstone func(key []byte) bool,
 	elideRangeTombstone func(start, end []byte) bool,
+	ineffectualSingleDeleteCallback func(userKey []byte),
+	singleDeleteInvariantViolationCallback func(userKey []byte),
 	formatVersion FormatMajorVersion,
 ) *compactionIter {
 	i := &compactionIter{
-		equal:               equal,
+		equal:                                  equal,
-		merge:               merge,
+		merge:                                  merge,
-		iter:                iter,
+		iter:                                   iter,
-		snapshots:           snapshots,
+		snapshots:                              snapshots,
-		frontiers:           frontiers{cmp: cmp},
+		frontiers:                              frontiers{cmp: cmp},
-		rangeDelFrag:        rangeDelFrag,
+		rangeDelFrag:                           rangeDelFrag,
-		rangeKeyFrag:        rangeKeyFrag,
+		rangeKeyFrag:                           rangeKeyFrag,
-		allowZeroSeqNum:     allowZeroSeqNum,
+		allowZeroSeqNum:                        allowZeroSeqNum,
-		elideTombstone:      elideTombstone,
+		elideTombstone:                         elideTombstone,
-		elideRangeTombstone: elideRangeTombstone,
+		elideRangeTombstone:                    elideRangeTombstone,
-		formatVersion:       formatVersion,
+		ineffectualSingleDeleteCallback:        ineffectualSingleDeleteCallback,
+		singleDeleteInvariantViolationCallback: singleDeleteInvariantViolationCallback,
+		formatVersion:                          formatVersion,
 	}
 	i.rangeDelFrag.Cmp = cmp
 	i.rangeDelFrag.Format = formatKey
@@ -333,7 +339,7 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 		return nil, nil
 	}
 
-	// Prior to this call to `Next()` we are in one of three situations with
+	// Prior to this call to `Next()` we are in one of four situations with
 	// respect to `iterKey` and related state:
 	//
 	// - `!skip && pos == iterPosNext`: `iterKey` is already at the next key.
@@ -342,12 +348,34 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 	//   snapshot stripe.
 	// - `skip && pos == iterPosCurForward`: We are at the key that has been returned.
 	//   To move forward we skip skippable entries in the stripe.
+	// - `skip && pos == iterPosNext && i.iterStripeChange == sameStripeNonSkippable`:
+	//    This case may occur when skipping within a snapshot stripe and we
+	//    encounter either:
+	//      a) an invalid key kind; The previous call will have returned
+	//         whatever key it was processing and deferred handling of the
+	//         invalid key to this invocation of Next(). We're responsible for
+	//         ignoring skip=true and falling into the invalid key kind case
+	//         down below.
+	//      b) an interleaved range delete; This is a wart of the current code
+	//         structure. While skipping within a snapshot stripe, a range
+	//         delete interleaved at its start key and sequence number
+	//         interrupts the sequence of point keys. After we return the range
+	//         delete to the caller, we need to pick up skipping at where we
+	//         left off, so we preserve skip=true.
+	//    TODO(jackson): This last case is confusing and can be removed if we
+	//    interleave range deletions at the maximal sequence number using the
+	//    keyspan interleaving iterator. This is the treatment given to range
+	//    keys today.
 	if i.pos == iterPosCurForward {
 		if i.skip {
 			i.skipInStripe()
 		} else {
 			i.nextInStripe()
 		}
+	} else if i.skip {
+		if i.iterStripeChange != sameStripeNonSkippable {
+			panic(errors.AssertionFailedf("compaction iterator has skip=true, but iterator is at iterPosNext"))
+		}
 	}
 
 	i.pos = iterPosCurForward
@@ -401,6 +429,10 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			return &i.key, i.value
 		}
 
+		// TODO(sumeer): we could avoid calling Covers if i.iterStripeChange ==
+		// sameStripeSameKey since that check has already been done in
+		// nextInStripeHelper. However, we also need to handle the case of
+		// CoversInvisibly below.
 		if cover := i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum); cover == keyspan.CoversVisibly {
 			// A pending range deletion deletes this key. Skip it.
 			i.saveKey()
@@ -429,7 +461,17 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 					// If we're at the last snapshot stripe and the tombstone
 					// can be elided skip skippable keys in the same stripe.
 					i.saveKey()
-					i.skipInStripe()
+					if i.key.Kind() == InternalKeyKindSingleDelete {
+						i.skipDueToSingleDeleteElision()
+					} else {
+						i.skipInStripe()
+						if !i.skip && i.iterStripeChange != newStripeNewKey {
+							panic(errors.AssertionFailedf("pebble: skipInStripe in last stripe disabled skip without advancing to new key"))
+						}
+					}
+					if i.iterStripeChange == newStripeSameKey {
+						panic(errors.AssertionFailedf("pebble: skipInStripe in last stripe found a new stripe within the same key"))
+					}
 					continue
 				} else {
 					// We're not at the last snapshot stripe, so the tombstone
@@ -456,8 +498,14 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			case InternalKeyKindSingleDelete:
 				if i.singleDeleteNext() {
 					return &i.key, i.value
+				} else if i.err != nil {
+					return nil, nil
 				}
 				continue
+
+			default:
+				panic(errors.AssertionFailedf(
+					"unexpected kind %s", redact.SafeString(i.iterKey.Kind().String())))
 			}
 
 		case InternalKeyKindSet, InternalKeyKindSetWithDelete:
@@ -467,6 +515,9 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			// preserving the original value, and potentially mutating the key
 			// kind.
 			i.setNext()
+			if i.err != nil {
+				return nil, nil
+			}
 			return &i.key, i.value
 
 		case InternalKeyKindMerge:
@@ -483,7 +534,15 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			if i.err == nil {
 				// includesBase is true whenever we've transformed the MERGE record
 				// into a SET.
-				includesBase := i.key.Kind() == InternalKeyKindSet
+				var includesBase bool
+				switch i.key.Kind() {
+				case InternalKeyKindSet, InternalKeyKindSetWithDelete:
+					includesBase = true
+				case InternalKeyKindMerge:
+				default:
+					panic(errors.AssertionFailedf(
+						"unexpected kind %s", redact.SafeString(i.key.Kind().String())))
+				}
 				i.value, needDelete, i.valueCloser, i.err = finishValueMerger(valueMerger, includesBase)
 			}
 			if i.err == nil {
@@ -512,6 +571,8 @@ func (i *compactionIter) Next() (*InternalKey, []byte) {
 			}
 			if i.err != nil {
 				i.valid = false
+				// TODO(sumeer): why is MarkCorruptionError only being called for
+				// MERGE?
 				i.err = base.MarkCorruptionError(i.err)
 			}
 			return nil, nil
@@ -551,10 +612,16 @@ func snapshotIndex(seq uint64, snapshots []uint64) (int, uint64) {
 	return index, snapshots[index]
 }
 
-// skipInStripe skips over skippable keys in the same stripe and user key.
+// skipInStripe skips over skippable keys in the same stripe and user key. It
+// may set i.err, in which case i.iterKey will be nil.
 func (i *compactionIter) skipInStripe() {
 	i.skip = true
+	// TODO(sumeer): we can avoid the overhead of calling i.rangeDelFrag.Covers,
+	// in this case of nextInStripe, since we are skipping all of them anyway.
 	for i.nextInStripe() == sameStripeSkippable {
+		if i.err != nil {
+			panic(i.err)
+		}
 	}
 	// Reset skip if we landed outside the original stripe. Otherwise, we landed
 	// in the same stripe on a non-skippable key. In that case we should preserve
@@ -592,11 +659,17 @@ const (
 // nextInStripe advances the iterator and returns one of the above const ints
 // indicating how its state changed.
 //
+// All sameStripeSkippable keys that are covered by a RANGEDEL will be skipped
+// and not returned.
+//
 // Calls to nextInStripe must be preceded by a call to saveKey to retain a
 // temporary reference to the original key, so that forward iteration can
 // proceed with a reference to the original key. Care should be taken to avoid
 // overwriting or mutating the saved key or value before they have been returned
 // to the caller of the exported function (i.e. the caller of Next, First, etc.)
+//
+// nextInStripe may set i.err, in which case the return value will be
+// newStripeNewKey, and i.iterKey will be nil.
 func (i *compactionIter) nextInStripe() stripeChangeType {
 	i.iterStripeChange = i.nextInStripeHelper()
 	return i.iterStripeChange
@@ -605,58 +678,72 @@ func (i *compactionIter) nextInStripe() stripeChangeType {
 // nextInStripeHelper is an internal helper for nextInStripe; callers should use
 // nextInStripe and not call nextInStripeHelper.
 func (i *compactionIter) nextInStripeHelper() stripeChangeType {
-	if !i.iterNext() {
+	origSnapshotIdx := i.curSnapshotIdx
-		return newStripeNewKey
+	for {
-	}
+		if !i.iterNext() {
-	key := i.iterKey
+			return newStripeNewKey
+		}
+		key := i.iterKey
 
-	// NB: The below conditional is an optimization to avoid a user key
+		if !i.equal(i.key.UserKey, key.UserKey) {
-	// comparison in many cases. Internal keys with the same user key are
+			i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
-	// ordered in (strictly) descending order by trailer. If the new key has a
+			return newStripeNewKey
-	// greater or equal trailer, or the previous key had a zero sequence number,
+		}
-	// the new key must have a new user key.
+
-	//
+		// If i.key and key have the same user key, then
-	// A couple things make these cases common:
+		//   1. i.key must not have had a zero sequence number (or it would've be the last
-	// - Sequence-number zeroing ensures ~all of the keys in L6 have a zero
+		//      key with its user key).
-	//   sequence number.
+		//   2. i.key must have a strictly larger sequence number
-	// - Ingested sstables' keys all adopt the same sequence number.
+		// There's an exception in that either key may be a range delete. Range
-	if i.keyTrailer <= base.InternalKeyZeroSeqnumMaxTrailer || key.Trailer >= i.keyTrailer {
+		// deletes may share a sequence number with a point key if the keys were
-		if invariants.Enabled && i.equal(i.key.UserKey, key.UserKey) {
+		// ingested together. Range keys may also share the sequence number if they
+		// were ingested, but range keys are interleaved into the compaction
+		// iterator's input iterator at the maximal sequence number so their
+		// original sequence number will not be observed here.
+		if prevSeqNum := base.SeqNumFromTrailer(i.keyTrailer); (prevSeqNum == 0 || prevSeqNum <= key.SeqNum()) &&
+			i.key.Kind() != InternalKeyKindRangeDelete && key.Kind() != InternalKeyKindRangeDelete {
 			prevKey := i.key
 			prevKey.Trailer = i.keyTrailer
-			panic(fmt.Sprintf("pebble: invariant violation: %s and %s out of order", key, prevKey))
+			panic(errors.AssertionFailedf("pebble: invariant violation: %s and %s out of order", prevKey, key))
 		}
+
 		i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
-		return newStripeNewKey
+		switch key.Kind() {
-	} else if !i.equal(i.key.UserKey, key.UserKey) {
+		case InternalKeyKindRangeDelete:
-		i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
+			// Range tombstones need to be exposed by the compactionIter to the upper level
-		return newStripeNewKey
+			// `compaction` object, so return them regardless of whether they are in the same
-	}
+			// snapshot stripe.
-	origSnapshotIdx := i.curSnapshotIdx
+			if i.curSnapshotIdx == origSnapshotIdx {
-	i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots)
+				return sameStripeNonSkippable
-	switch key.Kind() {
+			}
-	case InternalKeyKindRangeDelete:
+			return newStripeSameKey
-		// Range tombstones need to be exposed by the compactionIter to the upper level
+		case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
-		// `compaction` object, so return them regardless of whether they are in the same
+			// Range keys are interleaved at the max sequence number for a given user
-		// snapshot stripe.
+			// key, so we should not see any more range keys in this stripe.
-		if i.curSnapshotIdx == origSnapshotIdx {
+			panic("unreachable")
-			return sameStripeNonSkippable
+		case InternalKeyKindInvalid:
+			if i.curSnapshotIdx == origSnapshotIdx {
+				return sameStripeNonSkippable
+			}
+			return newStripeSameKey
+		case InternalKeyKindDelete, InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSingleDelete,
+			InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
+			// Fall through
+		default:
+			i.iterKey = nil
+			i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
+			i.valid = false
+			return newStripeNewKey
 		}
-		return newStripeSameKey
-	case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
-		// Range keys are interleaved at the max sequence number for a given user
-		// key, so we should not see any more range keys in this stripe.
-		panic("unreachable")
-	case InternalKeyKindInvalid:
 		if i.curSnapshotIdx == origSnapshotIdx {
-			return sameStripeNonSkippable
+			// Same snapshot.
+			if i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
+				continue
+			}
+			return sameStripeSkippable
 		}
 		return newStripeSameKey
 	}
-	if i.curSnapshotIdx == origSnapshotIdx {
-		return sameStripeSkippable
-	}
-	return newStripeSameKey
 }
 
 func (i *compactionIter) setNext() {
@@ -682,6 +769,10 @@ func (i *compactionIter) setNext() {
 
 	// Else, we continue to loop through entries in the stripe looking for a
 	// DEL. Note that we may stop *before* encountering a DEL, if one exists.
+	//
+	// NB: nextInStripe will skip sameStripeSkippable keys that are visibly
+	// covered by a RANGEDEL. This can include DELs -- this is fine since such
+	// DELs don't need to be combined with SET to make SETWITHDEL.
 	for {
 		switch i.nextInStripe() {
 		case newStripeNewKey, newStripeSameKey:
@@ -732,12 +823,16 @@ func (i *compactionIter) setNext() {
 			// We're still in the same stripe. If this is a
 			// DEL/SINGLEDEL/DELSIZED, we stop looking and emit a SETWITHDEL.
 			// Subsequent keys are eligible for skipping.
-			if i.iterKey.Kind() == InternalKeyKindDelete ||
+			switch i.iterKey.Kind() {
-				i.iterKey.Kind() == InternalKeyKindSingleDelete ||
+			case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
-				i.iterKey.Kind() == InternalKeyKindDeleteSized {
 				i.key.SetKind(InternalKeyKindSetWithDelete)
 				i.skip = true
 				return
+			case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
+				// Do nothing
+			default:
+				i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
+				i.valid = false
 			}
 		default:
 			panic("pebble: unexpected stripeChangeType: " + strconv.Itoa(int(i.iterStripeChange)))
@@ -757,13 +852,24 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
 			i.pos = iterPosNext
 			return i.iterStripeChange
 		}
+		if i.err != nil {
+			panic(i.err)
+		}
+		// NB: MERGE#10+RANGEDEL#9 stays a MERGE, since nextInStripe skips
+		// sameStripeSkippable keys that are visibly covered by a RANGEDEL. There
+		// may be MERGE#7 that is invisibly covered and will be preserved, but
+		// there is no risk that MERGE#10 and MERGE#7 will get merged in the
+		// future as the RANGEDEL still exists and will be used in user-facing
+		// reads that see MERGE#10, and will also eventually cause MERGE#7 to be
+		// deleted in a compaction.
 		key := i.iterKey
 		switch key.Kind() {
 		case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized:
 			// We've hit a deletion tombstone. Return everything up to this point and
 			// then skip entries until the next snapshot stripe. We change the kind
 			// of the result key to a Set so that it shadows keys in lower
-			// levels. That is, MERGE+DEL -> SET.
+			// levels. That is, MERGE+DEL -> SETWITHDEL.
+			//
 			// We do the same for SingleDelete since SingleDelete is only
 			// permitted (with deterministic behavior) for keys that have been
 			// set once since the last SingleDelete/Delete, so everything
@@ -776,21 +882,11 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
 			// single Set, and then merge in any following Sets, but that is
 			// complicated wrt code and unnecessary given the narrow permitted
 			// use of SingleDelete.
-			i.key.SetKind(InternalKeyKindSet)
+			i.key.SetKind(InternalKeyKindSetWithDelete)
 			i.skip = true
 			return sameStripeSkippable
 
 		case InternalKeyKindSet, InternalKeyKindSetWithDelete:
-			if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
-				// We change the kind of the result key to a Set so that it shadows
-				// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
-				// strictly necessary, but provides consistency with the behavior of
-				// MERGE+DEL.
-				i.key.SetKind(InternalKeyKindSet)
-				i.skip = true
-				return sameStripeSkippable
-			}
-
 			// We've hit a Set or SetWithDel value. Merge with the existing
 			// value and return. We change the kind of the resulting key to a
 			// Set so that it shadows keys in lower levels. That is:
@@ -805,16 +901,6 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
 			return sameStripeSkippable
 
 		case InternalKeyKindMerge:
-			if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly {
-				// We change the kind of the result key to a Set so that it shadows
-				// keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't
-				// strictly necessary, but provides consistency with the behavior of
-				// MERGE+DEL.
-				i.key.SetKind(InternalKeyKindSet)
-				i.skip = true
-				return sameStripeSkippable
-			}
-
 			// We've hit another Merge value. Merge with the existing value and
 			// continue looping.
 			i.err = valueMerger.MergeOlder(i.iterValue)
@@ -831,6 +917,16 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType {
 	}
 }
 
+// singleDeleteNext processes a SingleDelete point tombstone. A SingleDelete, or
+// SINGLEDEL, is unique in that it deletes exactly 1 internal key. It's a
+// performance optimization when the client knows a user key has not been
+// overwritten, allowing the elision of the tombstone earlier, avoiding write
+// amplification.
+//
+// singleDeleteNext returns a boolean indicating whether or not the caller
+// should yield the SingleDelete key to the consumer of the compactionIter. If
+// singleDeleteNext returns false, the caller may consume/elide the
+// SingleDelete.
 func (i *compactionIter) singleDeleteNext() bool {
 	// Save the current key.
 	i.saveKey()
@@ -839,26 +935,85 @@ func (i *compactionIter) singleDeleteNext() bool {
 
 	// Loop until finds a key to be passed to the next level.
 	for {
+		// If we find a key that can't be skipped, return true so that the
+		// caller yields the SingleDelete to the caller.
 		if i.nextInStripe() != sameStripeSkippable {
+			// This defers additional error checking regarding single delete
+			// invariants to the compaction where the keys with the same user key as
+			// the single delete are in the same stripe.
 			i.pos = iterPosNext
-			return true
+			return i.err == nil
 		}
-
+		if i.err != nil {
+			panic(i.err)
+		}
+		// INVARIANT: sameStripeSkippable.
 		key := i.iterKey
-		switch key.Kind() {
+		kind := key.Kind()
-		case InternalKeyKindDelete, InternalKeyKindMerge, InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
+		switch kind {
-			// We've hit a Delete, DeleteSized, Merge, SetWithDelete, transform
+		case InternalKeyKindDelete, InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized:
+			if (kind == InternalKeyKindDelete || kind == InternalKeyKindDeleteSized) &&
+				i.ineffectualSingleDeleteCallback != nil {
+				i.ineffectualSingleDeleteCallback(i.key.UserKey)
+			}
+			// We've hit a Delete, DeleteSized, SetWithDelete, transform
 			// the SingleDelete into a full Delete.
 			i.key.SetKind(InternalKeyKindDelete)
 			i.skip = true
 			return true
 
-		case InternalKeyKindSet:
+		case InternalKeyKindSet, InternalKeyKindMerge:
-			i.nextInStripe()
+			// This SingleDelete deletes the Set/Merge, and we can now elide the
+			// SingleDel as well. We advance past the Set and return false to
+			// indicate to the main compaction loop that we should NOT yield the
+			// current SingleDel key to the compaction loop.
+			//
+			// NB: singleDeleteNext was called with i.pos == iterPosCurForward, and
+			// after the call to nextInStripe, we are still at iterPosCurForward,
+			// since we are at the key after the Set/Merge that was single deleted.
+			change := i.nextInStripe()
+			switch change {
+			case sameStripeSkippable, newStripeSameKey:
+				// On the same user key.
+				nextKind := i.iterKey.Kind()
+				switch nextKind {
+				case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
+					if i.singleDeleteInvariantViolationCallback != nil {
+						// sameStripeSkippable keys returned by nextInStripe() are already
+						// known to not be covered by a RANGEDEL, so it is an invariant
+						// violation. The rare case is newStripeSameKey, where it is a
+						// violation if not covered by a RANGEDEL.
+						if change == sameStripeSkippable ||
+							i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) == keyspan.NoCover {
+							i.singleDeleteInvariantViolationCallback(i.key.UserKey)
+						}
+					}
+				case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete,
+					InternalKeyKindRangeDelete:
+				default:
+					panic(errors.AssertionFailedf(
+						"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
+				}
+			case sameStripeNonSkippable:
+				// No ability to check whether there is another Set/Merge below with
+				// the same user key.
+				//
+				// TODO(sumeer): once range deletions are interleaved at the maximal
+				// sequence number, this case will go away.
+			case newStripeNewKey:
+			default:
+				panic("unreachable")
+			}
 			i.valid = false
 			return false
 
 		case InternalKeyKindSingleDelete:
+			// Two single deletes met in a compaction. The first single delete is
+			// ineffectual.
+			if i.ineffectualSingleDeleteCallback != nil {
+				i.ineffectualSingleDeleteCallback(i.key.UserKey)
+			}
+			// Continue to apply the second single delete.
 			continue
 
 		default:
@@ -869,6 +1024,135 @@ func (i *compactionIter) singleDeleteNext() bool {
 	}
 }
 
+// skipDueToSingleDeleteElision is called when the SingleDelete is being
+// elided because it is in the final snapshot stripe and there are no keys
+// with the same user key in lower levels in the LSM (below the files in this
+// compaction).
+//
+// TODO(sumeer): the only difference between singleDeleteNext and
+// skipDueToSingleDeleteElision is the fact that the caller knows it will be
+// eliding the single delete in the latter case. There are some similar things
+// happening in both implementations. My first attempt at combining them into
+// a single method was hard to comprehend. Try again.
+func (i *compactionIter) skipDueToSingleDeleteElision() {
+	for {
+		stripeChange := i.nextInStripe()
+		if i.err != nil {
+			panic(i.err)
+		}
+		switch stripeChange {
+		case newStripeNewKey:
+			// The single delete is only now being elided, meaning it did not elide
+			// any keys earlier in its descent down the LSM. We stepped onto a new
+			// user key, meaning that even now at its moment of elision, it still
+			// hasn't elided any other keys. The single delete was ineffectual (a
+			// no-op).
+			if i.ineffectualSingleDeleteCallback != nil {
+				i.ineffectualSingleDeleteCallback(i.key.UserKey)
+			}
+			i.skip = false
+			return
+		case newStripeSameKey:
+			// This should be impossible. If we're eliding a single delete, we
+			// determined that the tombstone is in the final snapshot stripe, but we
+			// stepped into a new stripe of the same key.
+			panic(errors.AssertionFailedf("eliding single delete followed by same key in new stripe"))
+		case sameStripeNonSkippable:
+			// There's a key that we cannot skip. There are two possible cases:
+			//   a. The key is invalid. This is an error.
+			//   b. The key is a range deletion.
+			// The second case may also be an ineffectual single delete. However, it
+			// is possible that there is a SET that is at the same seqnum as the
+			// RANGEDEL, and so is not deleted by that RANGEDEL, and will be deleted
+			// by this single delete. So we cannot be certain that this is an
+			// ineffectual single delete.
+			//
+			// TODO(sumeer): the existing todo to interleave range deletions at the
+			// maximal sequence number will allow us to address this ambiguity.
+			//
+			// TODO(sumeer): by setting skip to true, the compactionIter is making a
+			// single delete stronger (like a del), which will hide bugs in the use of
+			// single delete.
+			i.skip = true
+			return
+		case sameStripeSkippable:
+			kind := i.iterKey.Kind()
+			switch kind {
+			case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete:
+				if i.ineffectualSingleDeleteCallback != nil {
+					i.ineffectualSingleDeleteCallback(i.key.UserKey)
+				}
+				switch kind {
+				case InternalKeyKindDelete, InternalKeyKindDeleteSized:
+					i.skipInStripe()
+					return
+				case InternalKeyKindSingleDelete:
+					// Repeat the same with this SingleDelete. We don't want to simply
+					// call skipInStripe(), since it increases the strength of the
+					// SingleDel, which hides bugs in the use of single delete.
+					continue
+				default:
+					panic(errors.AssertionFailedf(
+						"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
+				}
+			case InternalKeyKindSetWithDelete:
+				// The SingleDelete should behave like a Delete.
+				i.skipInStripe()
+				return
+			case InternalKeyKindSet, InternalKeyKindMerge:
+				// This SingleDelete deletes the Set/Merge, and we are eliding the
+				// SingleDel as well. Step to the next key (this is not deleted by the
+				// SingleDelete).
+				//
+				// NB: skipDueToSingleDeleteElision was called with i.pos ==
+				// iterPosCurForward, and after the call to nextInStripe, we are still
+				// at iterPosCurForward, since we are at the key after the Set/Merge
+				// that was single deleted.
+				change := i.nextInStripe()
+				if i.err != nil {
+					panic(i.err)
+				}
+				switch change {
+				case newStripeSameKey:
+					panic(errors.AssertionFailedf("eliding single delete followed by same key in new stripe"))
+				case newStripeNewKey:
+				case sameStripeSkippable:
+					// On the same key.
+					nextKind := i.iterKey.Kind()
+					switch nextKind {
+					case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
+						if i.singleDeleteInvariantViolationCallback != nil {
+							i.singleDeleteInvariantViolationCallback(i.key.UserKey)
+						}
+					case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete,
+						InternalKeyKindRangeDelete:
+					default:
+						panic(errors.AssertionFailedf(
+							"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
+					}
+				case sameStripeNonSkippable:
+					// No ability to check whether there is another Set/Merge below with
+					// the same user key.
+					//
+					// TODO(sumeer): once range deletions are interleaved at the maximal
+					// sequence number, this case will go away.
+				default:
+					panic("unreachable")
+				}
+				// Whether in same stripe or new stripe, this key is not consumed by
+				// the SingleDelete.
+				i.skip = false
+				return
+			default:
+				panic(errors.AssertionFailedf(
+					"unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind())))
+			}
+		default:
+			panic("unreachable")
+		}
+	}
+}
+
 // deleteSizedNext processes a DELSIZED point tombstone. Unlike ordinary DELs,
 // these tombstones carry a value that's a varint indicating the size of the
 // entry (len(key)+len(value)) that the tombstone is expected to delete.
@@ -894,8 +1178,11 @@ func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) {
 	// Loop through all the keys within this stripe that are skippable.
 	i.pos = iterPosNext
 	for i.nextInStripe() == sameStripeSkippable {
+		if i.err != nil {
+			panic(i.err)
+		}
 		switch i.iterKey.Kind() {
-		case InternalKeyKindDelete, InternalKeyKindDeleteSized:
+		case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete:
 			// We encountered a tombstone (DEL, or DELSIZED) that's deleted by
 			// the original DELSIZED tombstone. This can happen in two cases:
 			//
@@ -917,8 +1204,8 @@ func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) {
 			// We can differentiate these two cases by examining the length of
 			// the DELSIZED's value. A DELSIZED's value holds the size of both
 			// the user key and value that it intends to delete. For any user
-			// key with a length > 1, a DELSIZED that has not deleted a key must
+			// key with a length > 0, a DELSIZED that has not deleted a key must
-			// have a value with a length > 1.
+			// have a value with a length > 0.
 			//
 			// We treat both cases the same functionally, adopting the identity
 			// of the lower-sequence numbered tombstone. However in the second
@@ -930,20 +1217,28 @@ func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) {
 			}
 			i.valueBuf = append(i.valueBuf[:0], i.iterValue...)
 			i.value = i.valueBuf
-			if i.iterKey.Kind() == InternalKeyKindDelete {
+			if i.iterKey.Kind() != InternalKeyKindDeleteSized {
-				// Convert the DELSIZED to a DEL—The DEL we're eliding may not
+				// Convert the DELSIZED to a DEL—The DEL/SINGLEDEL we're eliding
-				// have deleted the key(s) it was intended to yet. The ordinary
+				// may not have deleted the key(s) it was intended to yet. The
-				// DEL compaction heuristics are better suited at that, plus we
+				// ordinary DEL compaction heuristics are better suited at that,
-				// don't want to count it as a missized DEL. We early exit in
+				// plus we don't want to count it as a missized DEL. We early
-				// this case, after skipping the remainder of the snapshot
+				// exit in this case, after skipping the remainder of the
-				// stripe.
+				// snapshot stripe.
-				i.key.SetKind(i.iterKey.Kind())
+				i.key.SetKind(InternalKeyKindDelete)
+				// NB: We skipInStripe now, rather than returning leaving
+				// i.skip=true and returning early, because Next() requires
+				// that i.skip=true only if i.iterPos = iterPosCurForward.
+				//
+				// Ignore any error caused by skipInStripe since it does not affect
+				// the key/value being returned here, and the next call to Next() will
+				// expose it.
 				i.skipInStripe()
 				return &i.key, i.value
 			}
 			// Continue, in case we uncover another DELSIZED or a key this
 			// DELSIZED deletes.
-		default:
+
+		case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete:
 			// If the DELSIZED is value-less, it already deleted the key that it
 			// was intended to delete. This is possible with a sequence like:
 			//
@@ -955,6 +1250,13 @@ func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) {
 			// the snapshot stripe and return.
 			if len(i.value) == 0 {
 				i.key.SetKind(InternalKeyKindDelete)
+				// NB: We skipInStripe now, rather than returning leaving
+				// i.skip=true and returning early, because Next() requires
+				// that i.skip=true only if i.iterPos = iterPosCurForward.
+				//
+				// Ignore any error caused by skipInStripe since it does not affect
+				// the key/value being returned here, and the next call to Next() will
+				// expose it.
 				i.skipInStripe()
 				return &i.key, i.value
 			}
@@ -982,13 +1284,28 @@ func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) {
 				// We opt for (4) under the rationale that we can't rely on the
 				// user-provided size for accuracy, so ordinary DEL heuristics
 				// are safer.
-				i.key.SetKind(InternalKeyKindDelete)
 				i.stats.countMissizedDels++
+				i.key.SetKind(InternalKeyKindDelete)
+				i.value = i.valueBuf[:0]
+				// NB: We skipInStripe now, rather than returning leaving
+				// i.skip=true and returning early, because Next() requires
+				// that i.skip=true only if i.iterPos = iterPosCurForward.
+				//
+				// Ignore any error caused by skipInStripe since it does not affect
+				// the key/value being returned here, and the next call to Next() will
+				// expose it.
+				i.skipInStripe()
+				return &i.key, i.value
 			}
 			// NB: We remove the value regardless of whether the key was sized
 			// appropriately. The size encoded is 'consumed' the first time it
 			// meets a key that it deletes.
 			i.value = i.valueBuf[:0]
+
+		default:
+			i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind()))
+			i.valid = false
+			return nil, nil
 		}
 	}
 	// Reset skip if we landed outside the original stripe. Otherwise, we landed
@@ -998,6 +1315,9 @@ func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) {
 	if i.iterStripeChange == newStripeNewKey || i.iterStripeChange == newStripeSameKey {
 		i.skip = false
 	}
+	if i.err != nil {
+		return nil, nil
+	}
 	return &i.key, i.value
 }
 

vendor/github.com/cockroachdb/pebble/compaction_picker.go

@@ -90,18 +90,38 @@ func (info compactionInfo) String() string {
 	return buf.String()
 }
 
-type sortCompactionLevelsDecreasingScore []candidateLevelInfo
+type sortCompactionLevelsByPriority []candidateLevelInfo
 
-func (s sortCompactionLevelsDecreasingScore) Len() int {
+func (s sortCompactionLevelsByPriority) Len() int {
 	return len(s)
 }
-func (s sortCompactionLevelsDecreasingScore) Less(i, j int) bool {
+
-	if s[i].score != s[j].score {
+// A level should be picked for compaction if the compensatedScoreRatio is >= the
-		return s[i].score > s[j].score
+// compactionScoreThreshold.
+const compactionScoreThreshold = 1
+
+// Less should return true if s[i] must be placed earlier than s[j] in the final
+// sorted list. The candidateLevelInfo for the level placed earlier is more likely
+// to be picked for a compaction.
+func (s sortCompactionLevelsByPriority) Less(i, j int) bool {
+	iShouldCompact := s[i].compensatedScoreRatio >= compactionScoreThreshold
+	jShouldCompact := s[j].compensatedScoreRatio >= compactionScoreThreshold
+	// Ordering is defined as decreasing on (shouldCompact, uncompensatedScoreRatio)
+	// where shouldCompact is 1 for true and 0 for false.
+	if iShouldCompact && !jShouldCompact {
+		return true
+	}
+	if !iShouldCompact && jShouldCompact {
+		return false
+	}
+
+	if s[i].uncompensatedScoreRatio != s[j].uncompensatedScoreRatio {
+		return s[i].uncompensatedScoreRatio > s[j].uncompensatedScoreRatio
 	}
 	return s[i].level < s[j].level
 }
-func (s sortCompactionLevelsDecreasingScore) Swap(i, j int) {
+
+func (s sortCompactionLevelsByPriority) Swap(i, j int) {
 	s[i], s[j] = s[j], s[i]
 }
 
@@ -151,13 +171,22 @@ func generateSublevelInfo(cmp base.Compare, levelFiles manifest.LevelSlice) []su
 	return levelSlices
 }
 
+// compactionPickerMetrics holds metrics related to the compaction picking process
+type compactionPickerMetrics struct {
+	// scores contains the compensatedScoreRatio from the candidateLevelInfo.
+	scores                      []float64
+	singleLevelOverlappingRatio float64
+	multiLevelOverlappingRatio  float64
+}
+
 // pickedCompaction contains information about a compaction that has already
 // been chosen, and is being constructed. Compaction construction info lives in
 // this struct, and is copied over into the compaction struct when that's
 // created.
 type pickedCompaction struct {
 	cmp Compare
-	// score of the chosen compaction. Taken from candidateLevelInfo.
+	// score of the chosen compaction. This is the same as the
+	// compensatedScoreRatio in the candidateLevelInfo.
 	score float64
 	// kind indicates the kind of compaction.
 	kind compactionKind
@@ -173,17 +202,12 @@ type pickedCompaction struct {
 	// extraLevels contain additional levels in between the input and output
 	// levels that get compacted in multi level compactions
 	extraLevels []*compactionLevel
-	// adjustedOutputLevel is the output level used for the purpose of
+	inputs      []compactionLevel
-	// determining the target output file size, overlap bytes, and expanded
+	// LBase at the time of compaction picking.
-	// bytes, taking into account the base level.
+	baseLevel int
-	adjustedOutputLevel int
-	inputs              []compactionLevel
 	// L0-specific compaction info. Set to a non-nil value for all compactions
 	// where startLevel == 0 that were generated by L0Sublevels.
 	lcf *manifest.L0CompactionFiles
-	// L0SublevelInfo is used for compactions out of L0. It is nil for all
-	// other compactions.
-	l0SublevelInfo []sublevelInfo
 	// maxOutputFileSize is the maximum size of an individual table created
 	// during compaction.
 	maxOutputFileSize uint64
@@ -195,9 +219,10 @@ type pickedCompaction struct {
 	// maxReadCompaction bytes, then we don't proceed with the compaction.
 	maxReadCompactionBytes uint64
 	// The boundaries of the input data.
-	smallest InternalKey
+	smallest      InternalKey
-	largest  InternalKey
+	largest       InternalKey
-	version  *version
+	version       *version
+	pickerMetrics compactionPickerMetrics
 }
 
 func defaultOutputLevel(startLevel, baseLevel int) int {
@@ -219,28 +244,35 @@ func newPickedCompaction(
 			startLevel, baseLevel))
 	}
 
-	adjustedOutputLevel := outputLevel
+	adjustedLevel := adjustedOutputLevel(outputLevel, baseLevel)
-	if adjustedOutputLevel > 0 {
-		// Output level is in the range [baseLevel,numLevels]. For the purpose of
-		// determining the target output file size, overlap bytes, and expanded
-		// bytes, we want to adjust the range to [1,numLevels].
-		adjustedOutputLevel = 1 + outputLevel - baseLevel
-	}
-
 	pc := &pickedCompaction{
 		cmp:                    opts.Comparer.Compare,
 		version:                cur,
+		baseLevel:              baseLevel,
 		inputs:                 []compactionLevel{{level: startLevel}, {level: outputLevel}},
-		adjustedOutputLevel:    adjustedOutputLevel,
+		maxOutputFileSize:      uint64(opts.Level(adjustedLevel).TargetFileSize),
-		maxOutputFileSize:      uint64(opts.Level(adjustedOutputLevel).TargetFileSize),
+		maxOverlapBytes:        maxGrandparentOverlapBytes(opts, adjustedLevel),
-		maxOverlapBytes:        maxGrandparentOverlapBytes(opts, adjustedOutputLevel),
+		maxReadCompactionBytes: maxReadCompactionBytes(opts, adjustedLevel),
-		maxReadCompactionBytes: maxReadCompactionBytes(opts, adjustedOutputLevel),
 	}
 	pc.startLevel = &pc.inputs[0]
 	pc.outputLevel = &pc.inputs[1]
 	return pc
 }
 
+// adjustedOutputLevel is the output level used for the purpose of
+// determining the target output file size, overlap bytes, and expanded
+// bytes, taking into account the base level.
+func adjustedOutputLevel(outputLevel int, baseLevel int) int {
+	adjustedOutputLevel := outputLevel
+	if adjustedOutputLevel > 0 {
+		// Output level is in the range [baseLevel, numLevels]. For the purpose of
+		// determining the target output file size, overlap bytes, and expanded
+		// bytes, we want to adjust the range to [1,numLevels].
+		adjustedOutputLevel = 1 + outputLevel - baseLevel
+	}
+	return adjustedOutputLevel
+}
+
 func newPickedCompactionFromL0(
 	lcf *manifest.L0CompactionFiles, opts *Options, vers *version, baseLevel int, isBase bool,
 ) *pickedCompaction {
@@ -273,7 +305,7 @@ func (pc *pickedCompaction) String() string {
 	var builder strings.Builder
 	builder.WriteString(fmt.Sprintf(`Score=%f, `, pc.score))
 	builder.WriteString(fmt.Sprintf(`Kind=%s, `, pc.kind))
-	builder.WriteString(fmt.Sprintf(`AdjustedOutputLevel=%d, `, pc.adjustedOutputLevel))
+	builder.WriteString(fmt.Sprintf(`AdjustedOutputLevel=%d, `, adjustedOutputLevel(pc.outputLevel.level, pc.baseLevel)))
 	builder.WriteString(fmt.Sprintf(`maxOutputFileSize=%d, `, pc.maxOutputFileSize))
 	builder.WriteString(fmt.Sprintf(`maxReadCompactionBytes=%d, `, pc.maxReadCompactionBytes))
 	builder.WriteString(fmt.Sprintf(`smallest=%s, `, pc.smallest))
@@ -283,7 +315,7 @@ func (pc *pickedCompaction) String() string {
 	builder.WriteString(fmt.Sprintf(`startlevel=%s, `, pc.startLevel))
 	builder.WriteString(fmt.Sprintf(`outputLevel=%s, `, pc.outputLevel))
 	builder.WriteString(fmt.Sprintf(`extraLevels=%s, `, pc.extraLevels))
-	builder.WriteString(fmt.Sprintf(`l0SublevelInfo=%s, `, pc.l0SublevelInfo))
+	builder.WriteString(fmt.Sprintf(`l0SublevelInfo=%s, `, pc.startLevel.l0SublevelInfo))
 	builder.WriteString(fmt.Sprintf(`lcf=%s`, pc.lcf))
 	return builder.String()
 }
@@ -297,13 +329,16 @@ func (pc *pickedCompaction) clone() *pickedCompaction {
 		cmp:                    pc.cmp,
 		score:                  pc.score,
 		kind:                   pc.kind,
-		adjustedOutputLevel:    pc.adjustedOutputLevel,
+		baseLevel:              pc.baseLevel,
 		maxOutputFileSize:      pc.maxOutputFileSize,
 		maxOverlapBytes:        pc.maxOverlapBytes,
 		maxReadCompactionBytes: pc.maxReadCompactionBytes,
 		smallest:               pc.smallest.Clone(),
 		largest:                pc.largest.Clone(),
 
+		// TODO(msbutler): properly clone picker metrics
+		pickerMetrics: pc.pickerMetrics,
+
 		// Both copies see the same manifest, therefore, it's ok for them to se
 		// share the same pc. version.
 		version: pc.version,
@@ -322,9 +357,11 @@ func (pc *pickedCompaction) clone() *pickedCompaction {
 		}
 	}
 
-	newPC.l0SublevelInfo = make([]sublevelInfo, len(pc.l0SublevelInfo))
+	if len(pc.startLevel.l0SublevelInfo) > 0 {
-	for i := range pc.l0SublevelInfo {
+		newPC.startLevel.l0SublevelInfo = make([]sublevelInfo, len(pc.startLevel.l0SublevelInfo))
-		newPC.l0SublevelInfo[i] = pc.l0SublevelInfo[i].Clone()
+		for i := range pc.startLevel.l0SublevelInfo {
+			newPC.startLevel.l0SublevelInfo[i] = pc.startLevel.l0SublevelInfo[i].Clone()
+		}
 	}
 	if pc.lcf != nil {
 		newPC.lcf = pc.lcf.Clone()
@@ -370,7 +407,7 @@ func (pc *pickedCompaction) setupInputs(
 	// growing a compaction results in a larger size, the original compaction
 	// is used instead.
 	maxExpandedBytes := expandedCompactionByteSizeLimit(
-		opts, pc.adjustedOutputLevel, diskAvailBytes,
+		opts, adjustedOutputLevel(pc.outputLevel.level, pc.baseLevel), diskAvailBytes,
 	)
 
 	// Expand the initial inputs to a clean cut.
@@ -465,7 +502,7 @@ func (pc *pickedCompaction) setupInputs(
 
 	if pc.startLevel.level == 0 {
 		// We don't change the input files for the compaction beyond this point.
-		pc.l0SublevelInfo = generateSublevelInfo(pc.cmp, pc.startLevel.files)
+		pc.startLevel.l0SublevelInfo = generateSublevelInfo(pc.cmp, pc.startLevel.files)
 	}
 
 	return true
@@ -528,8 +565,6 @@ func (pc *pickedCompaction) setupMultiLevelCandidate(opts *Options, diskAvailByt
 	pc.startLevel = &pc.inputs[0]
 	pc.extraLevels = []*compactionLevel{&pc.inputs[1]}
 	pc.outputLevel = &pc.inputs[2]
-
-	pc.adjustedOutputLevel++
 	return pc.setupInputs(opts, diskAvailBytes, pc.extraLevels[len(pc.extraLevels)-1])
 }
 
@@ -653,16 +688,23 @@ func newCompactionPicker(
 // Information about a candidate compaction level that has been identified by
 // the compaction picker.
 type candidateLevelInfo struct {
-	// The score of the level to be compacted, with compensated file sizes and
+	// The compensatedScore of the level after adjusting according to the other
-	// adjustments.
+	// levels' sizes. For L0, the compensatedScoreRatio is equivalent to the
-	score float64
+	// uncompensatedScoreRatio as we don't account for level size compensation in
-	// The original score of the level to be compacted, before adjusting
+	// L0.
-	// according to other levels' sizes.
+	compensatedScoreRatio float64
-	origScore float64
+	// The score of the level after accounting for level size compensation before
-	// The raw score of the level to be compacted, calculated using
+	// adjusting according to other levels' sizes. For L0, the compensatedScore
-	// uncompensated file sizes and without any adjustments.
+	// is equivalent to the uncompensatedScore as we don't account for level
-	rawScore float64
+	// size compensation in L0.
-	level    int
+	compensatedScore float64
+	// The score of the level to be compacted, calculated using uncompensated file
+	// sizes and without any adjustments.
+	uncompensatedScore float64
+	// uncompensatedScoreRatio is the uncompensatedScore adjusted according to
+	// the other levels' sizes.
+	uncompensatedScoreRatio float64
+	level                   int
 	// The level to compact to.
 	outputLevel int
 	// The file in level that will be compacted. Additional files may be
@@ -671,6 +713,10 @@ type candidateLevelInfo struct {
 	file manifest.LevelFile
 }
 
+func (c *candidateLevelInfo) shouldCompact() bool {
+	return c.compensatedScoreRatio >= compactionScoreThreshold
+}
+
 func fileCompensation(f *fileMetadata) uint64 {
 	return uint64(f.Stats.PointDeletionsBytesEstimate) + f.Stats.RangeDeletionsBytesEstimate
 }
@@ -748,7 +794,7 @@ var _ compactionPicker = &compactionPickerByScore{}
 func (p *compactionPickerByScore) getScores(inProgress []compactionInfo) [numLevels]float64 {
 	var scores [numLevels]float64
 	for _, info := range p.calculateLevelScores(inProgress) {
-		scores[info.level] = info.score
+		scores[info.level] = info.compensatedScoreRatio
 	}
 	return scores
 }
@@ -967,68 +1013,83 @@ func (p *compactionPickerByScore) calculateLevelScores(
 		scores[i].level = i
 		scores[i].outputLevel = i + 1
 	}
+	l0UncompensatedScore := calculateL0UncompensatedScore(p.vers, p.opts, inProgressCompactions)
 	scores[0] = candidateLevelInfo{
-		outputLevel: p.baseLevel,
+		outputLevel:        p.baseLevel,
-		score:       calculateL0Score(p.vers, p.opts, inProgressCompactions),
+		uncompensatedScore: l0UncompensatedScore,
+		compensatedScore:   l0UncompensatedScore, /* No level size compensation for L0 */
 	}
 	sizeAdjust := calculateSizeAdjust(inProgressCompactions)
 	for level := 1; level < numLevels; level++ {
 		compensatedLevelSize := levelCompensatedSize(p.vers.Levels[level]) + sizeAdjust[level].compensated()
-		scores[level].score = float64(compensatedLevelSize) / float64(p.levelMaxBytes[level])
+		scores[level].compensatedScore = float64(compensatedLevelSize) / float64(p.levelMaxBytes[level])
-		scores[level].origScore = scores[level].score
+		scores[level].uncompensatedScore = float64(p.vers.Levels[level].Size()+sizeAdjust[level].actual()) / float64(p.levelMaxBytes[level])
-
-		// In addition to the compensated score, we calculate a separate score
-		// that uses actual file sizes, not compensated sizes. This is used
-		// during score smoothing down below to prevent excessive
-		// prioritization of reclaiming disk space.
-		scores[level].rawScore = float64(p.vers.Levels[level].Size()+sizeAdjust[level].actual()) / float64(p.levelMaxBytes[level])
 	}
 
-	// Adjust each level's score by the score of the next level. If the next
+	// Adjust each level's {compensated, uncompensated}Score by the uncompensatedScore
-	// level has a high score, and is thus a priority for compaction, this
+	// of the next level to get a {compensated, uncompensated}ScoreRatio. If the
-	// reduces the priority for compacting the current level. If the next level
+	// next level has a high uncompensatedScore, and is thus a priority for compaction,
-	// has a low score (i.e. it is below its target size), this increases the
+	// this reduces the priority for compacting the current level. If the next level
-	// priority for compacting the current level.
+	// has a low uncompensatedScore (i.e. it is below its target size), this increases
+	// the priority for compacting the current level.
 	//
 	// The effect of this adjustment is to help prioritize compactions in lower
-	// levels. The following shows the new score and original score. In this
+	// levels. The following example shows the compensatedScoreRatio and the
-	// scenario, L0 has 68 sublevels. L3 (a.k.a. Lbase) is significantly above
+	// compensatedScore. In this scenario, L0 has 68 sublevels. L3 (a.k.a. Lbase)
-	// its target size. The original score prioritizes compactions from those two
+	// is significantly above its target size. The original score prioritizes
-	// levels, but doing so ends up causing a future problem: data piles up in
+	// compactions from those two levels, but doing so ends up causing a future
-	// the higher levels, starving L5->L6 compactions, and to a lesser degree
+	// problem: data piles up in the higher levels, starving L5->L6 compactions,
-	// starving L4->L5 compactions.
+	// and to a lesser degree starving L4->L5 compactions.
 	//
-	//        adjusted   original
+	// Note that in the example shown there is no level size compensation so the
-	//           score      score       size   max-size
+	// compensatedScore and the uncompensatedScore is the same for each level.
-	//   L0        3.2       68.0      2.2 G          -
+	//
-	//   L3        3.2       21.1      1.3 G       64 M
+	//        compensatedScoreRatio   compensatedScore   uncompensatedScore   size   max-size
-	//   L4        3.4        6.7      3.1 G      467 M
+	//   L0                     3.2               68.0                 68.0  2.2 G          -
-	//   L5        3.4        2.0      6.6 G      3.3 G
+	//   L3                     3.2               21.1                 21.1  1.3 G       64 M
-	//   L6        0.6        0.6       14 G       24 G
+	//   L4                     3.4                6.7                  6.7  3.1 G      467 M
+	//   L5                     3.4                2.0                  2.0  6.6 G      3.3 G
+	//   L6                     0.6                0.6                  0.6   14 G       24 G
 	var prevLevel int
 	for level := p.baseLevel; level < numLevels; level++ {
-		if scores[prevLevel].score >= 1 {
+		// The compensated scores, and uncompensated scores will be turned into
-			// Avoid absurdly large scores by placing a floor on the score that we'll
+		// ratios as they're adjusted according to other levels' sizes.
-			// adjust a level by. The value of 0.01 was chosen somewhat arbitrarily
+		scores[prevLevel].compensatedScoreRatio = scores[prevLevel].compensatedScore
-			const minScore = 0.01
+		scores[prevLevel].uncompensatedScoreRatio = scores[prevLevel].uncompensatedScore
-			if scores[level].rawScore >= minScore {
+
-				scores[prevLevel].score /= scores[level].rawScore
+		// Avoid absurdly large scores by placing a floor on the score that we'll
+		// adjust a level by. The value of 0.01 was chosen somewhat arbitrarily.
+		const minScore = 0.01
+		if scores[prevLevel].compensatedScoreRatio >= compactionScoreThreshold {
+			if scores[level].uncompensatedScore >= minScore {
+				scores[prevLevel].compensatedScoreRatio /= scores[level].uncompensatedScore
 			} else {
-				scores[prevLevel].score /= minScore
+				scores[prevLevel].compensatedScoreRatio /= minScore
+			}
+		}
+		if scores[prevLevel].uncompensatedScoreRatio >= compactionScoreThreshold {
+			if scores[level].uncompensatedScore >= minScore {
+				scores[prevLevel].uncompensatedScoreRatio /= scores[level].uncompensatedScore
+			} else {
+				scores[prevLevel].uncompensatedScoreRatio /= minScore
 			}
 		}
 		prevLevel = level
 	}
+	// Set the score ratios for the lowest level.
+	// INVARIANT: prevLevel == numLevels-1
+	scores[prevLevel].compensatedScoreRatio = scores[prevLevel].compensatedScore
+	scores[prevLevel].uncompensatedScoreRatio = scores[prevLevel].uncompensatedScore
 
-	sort.Sort(sortCompactionLevelsDecreasingScore(scores[:]))
+	sort.Sort(sortCompactionLevelsByPriority(scores[:]))
 	return scores
 }
 
-// calculateL0Score calculates a float score representing the relative priority
+// calculateL0UncompensatedScore calculates a float score representing the
-// of compacting L0. Level L0 is special in that files within L0 may overlap one
+// relative priority of compacting L0. Level L0 is special in that files within
-// another, so a different set of heuristics that take into account
+// L0 may overlap one another, so a different set of heuristics that take into
-// read amplification apply.
+// account read amplification apply.
-func calculateL0Score(
+func calculateL0UncompensatedScore(
 	vers *version, opts *Options, inProgressCompactions []compactionInfo,
 ) float64 {
 	// Use the sublevel count to calculate the score. The base vs intra-L0
@@ -1231,8 +1292,9 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact
 			if pc.startLevel.level == info.level {
 				marker = "*"
 			}
-			fmt.Fprintf(&buf, "  %sL%d: %5.1f  %5.1f  %5.1f %8s  %8s",
+			fmt.Fprintf(&buf, "  %sL%d: %5.1f  %5.1f  %5.1f  %5.1f %8s  %8s",
-				marker, info.level, info.score, info.origScore, info.rawScore,
+				marker, info.level, info.compensatedScoreRatio, info.compensatedScore,
+				info.uncompensatedScoreRatio, info.uncompensatedScore,
 				humanize.Bytes.Int64(int64(totalCompensatedSize(
 					p.vers.Levels[info.level].Iter(),
 				))),
@@ -1262,12 +1324,12 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact
 			pc.startLevel.level, pc.outputLevel.level, buf.String())
 	}
 
-	// Check for a score-based compaction. "scores" has been sorted in order of
+	// Check for a score-based compaction. candidateLevelInfos are first sorted
-	// decreasing score. For each level with a score >= 1, we attempt to find a
+	// by whether they should be compacted, so if we find a level which shouldn't
-	// compaction anchored at at that level.
+	// be compacted, we can break early.
 	for i := range scores {
 		info := &scores[i]
-		if info.score < 1 {
+		if !info.shouldCompact() {
 			break
 		}
 		if info.level == numLevels-1 {
@@ -1279,7 +1341,8 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact
 			// Fail-safe to protect against compacting the same sstable
 			// concurrently.
 			if pc != nil && !inputRangeAlreadyCompacting(env, pc) {
-				pc.score = info.score
+				p.addScoresToPickedCompactionMetrics(pc, scores)
+				pc.score = info.compensatedScoreRatio
 				// TODO(bananabrick): Create an EventListener for logCompaction.
 				if false {
 					logCompaction(pc)
@@ -1299,7 +1362,8 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact
 		pc := pickAutoLPositive(env, p.opts, p.vers, *info, p.baseLevel, p.levelMaxBytes)
 		// Fail-safe to protect against compacting the same sstable concurrently.
 		if pc != nil && !inputRangeAlreadyCompacting(env, pc) {
-			pc.score = info.score
+			p.addScoresToPickedCompactionMetrics(pc, scores)
+			pc.score = info.compensatedScoreRatio
 			// TODO(bananabrick): Create an EventListener for logCompaction.
 			if false {
 				logCompaction(pc)
@@ -1360,6 +1424,30 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact
 	return nil
 }
 
+func (p *compactionPickerByScore) addScoresToPickedCompactionMetrics(
+	pc *pickedCompaction, candInfo [numLevels]candidateLevelInfo,
+) {
+
+	// candInfo is sorted by score, not by compaction level.
+	infoByLevel := [numLevels]candidateLevelInfo{}
+	for i := range candInfo {
+		level := candInfo[i].level
+		infoByLevel[level] = candInfo[i]
+	}
+	// Gather the compaction scores for the levels participating in the compaction.
+	pc.pickerMetrics.scores = make([]float64, len(pc.inputs))
+	inputIdx := 0
+	for i := range infoByLevel {
+		if pc.inputs[inputIdx].level == infoByLevel[i].level {
+			pc.pickerMetrics.scores[inputIdx] = infoByLevel[i].compensatedScoreRatio
+			inputIdx++
+		}
+		if inputIdx == len(pc.inputs) {
+			break
+		}
+	}
+}
+
 // elisionOnlyAnnotator implements the manifest.Annotator interface,
 // annotating B-Tree nodes with the *fileMetadata of a file meeting the
 // obsolete keys criteria for an elision-only compaction within the subtree.
@@ -1558,7 +1646,7 @@ func (p *compactionPickerByScore) pickRewriteCompaction(env compactionEnv) (pc *
 		// Fail-safe to protect against compacting the same sstable concurrently.
 		if !inputRangeAlreadyCompacting(env, pc) {
 			if pc.startLevel.level == 0 {
-				pc.l0SublevelInfo = generateSublevelInfo(pc.cmp, pc.startLevel.files)
+				pc.startLevel.l0SublevelInfo = generateSublevelInfo(pc.cmp, pc.startLevel.files)
 			}
 			return pc
 		}
@@ -1605,33 +1693,113 @@ func pickAutoLPositive(
 
 // maybeAddLevel maybe adds a level to the picked compaction.
 func (pc *pickedCompaction) maybeAddLevel(opts *Options, diskAvailBytes uint64) *pickedCompaction {
+	pc.pickerMetrics.singleLevelOverlappingRatio = pc.overlappingRatio()
 	if pc.outputLevel.level == numLevels-1 {
 		// Don't add a level if the current output level is in L6
 		return pc
 	}
+	if !opts.Experimental.MultiLevelCompactionHeuristic.allowL0() && pc.startLevel.level == 0 {
+		return pc
+	}
 	if pc.compactionSize() > expandedCompactionByteSizeLimit(
-		opts, pc.adjustedOutputLevel, diskAvailBytes) {
+		opts, adjustedOutputLevel(pc.outputLevel.level, pc.baseLevel), diskAvailBytes) {
 		// Don't add a level if the current compaction exceeds the compaction size limit
 		return pc
 	}
-	return opts.Experimental.MultiLevelCompactionHueristic.pick(pc, opts, diskAvailBytes)
+	return opts.Experimental.MultiLevelCompactionHeuristic.pick(pc, opts, diskAvailBytes)
 }
 
 // MultiLevelHeuristic evaluates whether to add files from the next level into the compaction.
 type MultiLevelHeuristic interface {
-	// evaluate returns the preferred compaction.
+	// Evaluate returns the preferred compaction.
 	pick(pc *pickedCompaction, opts *Options, diskAvailBytes uint64) *pickedCompaction
+
+	// Returns if the heuristic allows L0 to be involved in ML compaction
+	allowL0() bool
 }
 
 // NoMultiLevel will never add an additional level to the compaction.
 type NoMultiLevel struct{}
 
+var _ MultiLevelHeuristic = (*NoMultiLevel)(nil)
+
 func (nml NoMultiLevel) pick(
 	pc *pickedCompaction, opts *Options, diskAvailBytes uint64,
 ) *pickedCompaction {
 	return pc
 }
 
+func (nml NoMultiLevel) allowL0() bool {
+	return false
+}
+
+func (pc *pickedCompaction) predictedWriteAmp() float64 {
+	var bytesToCompact uint64
+	var higherLevelBytes uint64
+	for i := range pc.inputs {
+		levelSize := pc.inputs[i].files.SizeSum()
+		bytesToCompact += levelSize
+		if i != len(pc.inputs)-1 {
+			higherLevelBytes += levelSize
+		}
+	}
+	return float64(bytesToCompact) / float64(higherLevelBytes)
+}
+
+func (pc *pickedCompaction) overlappingRatio() float64 {
+	var higherLevelBytes uint64
+	var lowestLevelBytes uint64
+	for i := range pc.inputs {
+		levelSize := pc.inputs[i].files.SizeSum()
+		if i == len(pc.inputs)-1 {
+			lowestLevelBytes += levelSize
+			continue
+		}
+		higherLevelBytes += levelSize
+	}
+	return float64(lowestLevelBytes) / float64(higherLevelBytes)
+}
+
+// WriteAmpHeuristic defines a multi level compaction heuristic which will add
+// an additional level to the picked compaction if it reduces predicted write
+// amp of the compaction + the addPropensity constant.
+type WriteAmpHeuristic struct {
+	// addPropensity is a constant that affects the propensity to conduct multilevel
+	// compactions. If positive, a multilevel compaction may get picked even if
+	// the single level compaction has lower write amp, and vice versa.
+	AddPropensity float64
+
+	// AllowL0 if true, allow l0 to be involved in a ML compaction.
+	AllowL0 bool
+}
+
+var _ MultiLevelHeuristic = (*WriteAmpHeuristic)(nil)
+
+// TODO(msbutler): microbenchmark the extent to which multilevel compaction
+// picking slows down the compaction picking process.  This should be as fast as
+// possible since Compaction-picking holds d.mu, which prevents WAL rotations,
+// in-progress flushes and compactions from completing, etc. Consider ways to
+// deduplicate work, given that setupInputs has already been called.
+func (wa WriteAmpHeuristic) pick(
+	pcOrig *pickedCompaction, opts *Options, diskAvailBytes uint64,
+) *pickedCompaction {
+	pcMulti := pcOrig.clone()
+	if !pcMulti.setupMultiLevelCandidate(opts, diskAvailBytes) {
+		return pcOrig
+	}
+	picked := pcOrig
+	if pcMulti.predictedWriteAmp() <= pcOrig.predictedWriteAmp()+wa.AddPropensity {
+		picked = pcMulti
+	}
+	// Regardless of what compaction was picked, log the multilevelOverlapping ratio.
+	picked.pickerMetrics.multiLevelOverlappingRatio = pcMulti.overlappingRatio()
+	return picked
+}
+
+func (wa WriteAmpHeuristic) allowL0() bool {
+	return wa.AllowL0
+}
+
 // Helper method to pick compactions originating from L0. Uses information about
 // sublevels to generate a compaction.
 func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (pc *pickedCompaction) {
@@ -1653,7 +1821,7 @@ func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (pc
 		if pc.startLevel.files.Empty() {
 			opts.Logger.Fatalf("empty compaction chosen")
 		}
-		return pc
+		return pc.maybeAddLevel(opts, env.diskAvailBytes)
 	}
 
 	// Couldn't choose a base compaction. Try choosing an intra-L0

vendor/github.com/cockroachdb/pebble/db.go

@@ -18,6 +18,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/arenaskl"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invalidating"
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/manifest"
@@ -315,9 +316,6 @@ type DB struct {
 	closedCh chan struct{}
 
 	cleanupManager *cleanupManager
-	// testingAlwaysWaitForCleanup is set by some tests to force waiting for
-	// obsolete file deletion (to make events deterministic).
-	testingAlwaysWaitForCleanup bool
 
 	// During an iterator close, we may asynchronously schedule read compactions.
 	// We want to wait for those goroutines to finish, before closing the DB.
@@ -558,9 +556,7 @@ func (d *DB) getInternal(key []byte, b *Batch, s *Snapshot) ([]byte, io.Closer,
 	get := &buf.get
 	*get = getIter{
 		logger:   d.opts.Logger,
-		cmp:      d.cmp,
+		comparer: d.opts.Comparer,
-		equal:    d.equal,
-		split:    d.split,
 		newIters: d.newIters,
 		snapshot: seqNum,
 		key:      key,
@@ -808,6 +804,9 @@ func (d *DB) applyInternal(batch *Batch, opts *WriteOptions, noSyncWait bool) er
 	if err := d.closed.Load(); err != nil {
 		panic(err)
 	}
+	if batch.committing {
+		panic("pebble: batch already committing")
+	}
 	if batch.applied.Load() {
 		panic("pebble: batch already applied")
 	}
@@ -838,12 +837,19 @@ func (d *DB) applyInternal(batch *Batch, opts *WriteOptions, noSyncWait bool) er
 		}
 		// TODO(jackson): Assert that all range key operands are suffixless.
 	}
+	batch.committing = true
 
 	if batch.db == nil {
-		batch.refreshMemTableSize()
+		if err := batch.refreshMemTableSize(); err != nil {
+			return err
+		}
 	}
 	if batch.memTableSize >= d.largeBatchThreshold {
-		batch.flushable = newFlushableBatch(batch, d.opts.Comparer)
+		var err error
+		batch.flushable, err = newFlushableBatch(batch, d.opts.Comparer)
+		if err != nil {
+			return err
+		}
 	}
 	if err := d.commit.Commit(batch, sync, noSyncWait); err != nil {
 		// There isn't much we can do on an error here. The commit pipeline will be
@@ -986,15 +992,14 @@ var iterAllocPool = sync.Pool{
 //   - No snapshot: All fields are zero values.
 //   - Classic snapshot: Only `seqNum` is set. The latest readState will be used
 //     and the specified seqNum will be used as the snapshot seqNum.
-//   - EventuallyFileOnlySnapshot (EFOS) in pre-file-only state: `readState` and
+//   - EventuallyFileOnlySnapshot (EFOS) behaving as a classic snapshot. Only
-//     `seqNum` are set. The specified readState is used with the snapshot
+//     the `seqNum` is set. The latest readState will be used
-//     sequence number.
+//     and the specified seqNum will be used as the snapshot seqNum.
-//   - EFOS in file-only state: Only `seqNum` and `vers` are set. No readState
+//   - EFOS in file-only state: Only `seqNum` and `vers` are set. All the
-//     is referenced, as all the relevant SSTs are referenced by the *version.
+//     relevant SSTs are referenced by the *version.
 type snapshotIterOpts struct {
-	seqNum    uint64
+	seqNum uint64
-	readState *readState
+	vers   *version
-	vers      *version
 }
 
 // newIter constructs a new iterator, merging in batch iterators as an extra
@@ -1027,12 +1032,10 @@ func (d *DB) newIter(
 	// Grab and reference the current readState. This prevents the underlying
 	// files in the associated version from being deleted if there is a current
 	// compaction. The readState is unref'd by Iterator.Close().
-	readState := sOpts.readState
+	var readState *readState
-	if readState == nil && sOpts.vers == nil {
+	if sOpts.vers == nil {
 		// NB: loadReadState() calls readState.ref().
 		readState = d.loadReadState()
-	} else if readState != nil {
-		readState.ref()
 	} else {
 		// s.vers != nil
 		sOpts.vers.Ref()
@@ -1149,6 +1152,7 @@ func finishInitializingIter(ctx context.Context, buf *iterAlloc) *Iterator {
 				},
 			}
 			dbi.iter = &dbi.lazyCombinedIter
+			dbi.iter = invalidating.MaybeWrapIfInvariants(dbi.iter)
 		} else {
 			dbi.lazyCombinedIter.combinedIterState = combinedIterState{
 				initialized: true,
@@ -1249,11 +1253,9 @@ func (d *DB) newInternalIter(sOpts snapshotIterOpts, o *scanInternalOptions) *sc
 	// Grab and reference the current readState. This prevents the underlying
 	// files in the associated version from being deleted if there is a current
 	// compaction. The readState is unref'd by Iterator.Close().
-	readState := sOpts.readState
+	var readState *readState
-	if readState == nil && sOpts.vers == nil {
+	if sOpts.vers == nil {
 		readState = d.loadReadState()
-	} else if readState != nil {
-		readState.ref()
 	}
 	if sOpts.vers != nil {
 		sOpts.vers.Ref()
@@ -1421,12 +1423,12 @@ func (i *Iterator) constructPointIter(
 	addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Level) {
 		li := &levels[levelsIndex]
 
-		li.init(
+		li.init(ctx, i.opts, &i.comparer, i.newIters, files, level, internalOpts)
-			ctx, i.opts, i.comparer.Compare, i.comparer.Split, i.newIters, files, level, internalOpts)
 		li.initRangeDel(&mlevels[mlevelsIndex].rangeDelIter)
 		li.initBoundaryContext(&mlevels[mlevelsIndex].levelIterBoundaryContext)
 		li.initCombinedIterState(&i.lazyCombinedIter.combinedIterState)
-		mlevels[mlevelsIndex].iter = li
+		mlevels[mlevelsIndex].levelIter = li
+		mlevels[mlevelsIndex].iter = invalidating.MaybeWrapIfInvariants(li)
 
 		levelsIndex++
 		mlevelsIndex++
@@ -1452,7 +1454,7 @@ func (i *Iterator) constructPointIter(
 	buf.merging.snapshot = i.seqNum
 	buf.merging.batchSnapshot = i.batchSeqNum
 	buf.merging.combinedIterState = &i.lazyCombinedIter.combinedIterState
-	i.pointIter = &buf.merging
+	i.pointIter = invalidating.MaybeWrapIfInvariants(&buf.merging)
 	i.merging = &buf.merging
 }
 
@@ -1968,6 +1970,17 @@ func (d *DB) Metrics() *Metrics {
 
 	d.mu.versions.logLock()
 	metrics.private.manifestFileSize = uint64(d.mu.versions.manifest.Size())
+	metrics.Table.BackingTableCount = uint64(len(d.mu.versions.backingState.fileBackingMap))
+	metrics.Table.BackingTableSize = d.mu.versions.backingState.fileBackingSize
+	if invariants.Enabled {
+		var totalSize uint64
+		for _, backing := range d.mu.versions.backingState.fileBackingMap {
+			totalSize += backing.Size
+		}
+		if totalSize != metrics.Table.BackingTableSize {
+			panic("pebble: invalid backing table size accounting")
+		}
+	}
 	d.mu.versions.logUnlock()
 
 	metrics.LogWriter.FsyncLatency = d.mu.log.metrics.fsyncLatency
@@ -2746,14 +2759,40 @@ func (d *DB) SetCreatorID(creatorID uint64) error {
 
 // KeyStatistics keeps track of the number of keys that have been pinned by a
 // snapshot as well as counts of the different key kinds in the lsm.
+//
+// One way of using the accumulated stats, when we only have sets and dels,
+// and say the counts are represented as del_count, set_count,
+// del_latest_count, set_latest_count, snapshot_pinned_count.
+//
+//   - del_latest_count + set_latest_count is the set of unique user keys
+//     (unique).
+//
+//   - set_latest_count is the set of live unique user keys (live_unique).
+//
+//   - Garbage is del_count + set_count - live_unique.
+//
+//   - If everything were in the LSM, del_count+set_count-snapshot_pinned_count
+//     would also be the set of unique user keys (note that
+//     snapshot_pinned_count is counting something different -- see comment below).
+//     But snapshot_pinned_count only counts keys in the LSM so the excess here
+//     must be keys in memtables.
 type KeyStatistics struct {
-	// when a compaction determines a key is obsolete, but cannot elide the key
+	// TODO(sumeer): the SnapshotPinned* are incorrect in that these older
-	// because it's required by an open snapshot.
+	// versions can be in a different level. Either fix the accounting or
+	// rename these fields.
+
+	// SnapshotPinnedKeys represents obsolete keys that cannot be elided during
+	// a compaction, because they are required by an open snapshot.
 	SnapshotPinnedKeys int
-	// the total number of bytes of all snapshot pinned keys.
+	// SnapshotPinnedKeysBytes is the total number of bytes of all snapshot
+	// pinned keys.
 	SnapshotPinnedKeysBytes uint64
-	// Note: these fields are currently only populated for point keys (including range deletes).
+	// KindsCount is the count for each kind of key. It includes point keys,
+	// range deletes and range keys.
 	KindsCount [InternalKeyKindMax + 1]int
+	// LatestKindsCount is the count for each kind of key when it is the latest
+	// kind for a user key. It is only populated for point keys.
+	LatestKindsCount [InternalKeyKindMax + 1]int
 }
 
 // LSMKeyStatistics is used by DB.ScanStatistics.
@@ -2798,7 +2837,8 @@ func (d *DB) ScanStatistics(
 			// pinned by a snapshot.
 			size := uint64(key.Size())
 			kind := key.Kind()
-			if iterInfo.Kind == IteratorLevelLSM && d.equal(prevKey.UserKey, key.UserKey) {
+			sameKey := d.equal(prevKey.UserKey, key.UserKey)
+			if iterInfo.Kind == IteratorLevelLSM && sameKey {
 				stats.Levels[iterInfo.Level].SnapshotPinnedKeys++
 				stats.Levels[iterInfo.Level].SnapshotPinnedKeysBytes += size
 				stats.Accumulated.SnapshotPinnedKeys++
@@ -2807,6 +2847,12 @@ func (d *DB) ScanStatistics(
 			if iterInfo.Kind == IteratorLevelLSM {
 				stats.Levels[iterInfo.Level].KindsCount[kind]++
 			}
+			if !sameKey {
+				if iterInfo.Kind == IteratorLevelLSM {
+					stats.Levels[iterInfo.Level].LatestKindsCount[kind]++
+				}
+				stats.Accumulated.LatestKindsCount[kind]++
+			}
 
 			stats.Accumulated.KindsCount[kind]++
 			prevKey.CopyFrom(*key)

vendor/github.com/cockroachdb/pebble/event.go

@@ -39,10 +39,11 @@ func formatFileNums(tables []TableInfo) string {
 	return buf.String()
 }
 
-// LevelInfo contains info pertaining to a partificular level.
+// LevelInfo contains info pertaining to a particular level.
 type LevelInfo struct {
 	Level  int
 	Tables []TableInfo
+	Score  float64
 }
 
 func (i LevelInfo) String() string {
@@ -51,8 +52,11 @@ func (i LevelInfo) String() string {
 
 // SafeFormat implements redact.SafeFormatter.
 func (i LevelInfo) SafeFormat(w redact.SafePrinter, _ rune) {
-	w.Printf("L%d [%s] (%s)", redact.Safe(i.Level), redact.Safe(formatFileNums(i.Tables)),
+	w.Printf("L%d [%s] (%s) Score=%.2f",
-		redact.Safe(humanize.Bytes.Uint64(tablesTotalSize(i.Tables))))
+		redact.Safe(i.Level),
+		redact.Safe(formatFileNums(i.Tables)),
+		redact.Safe(humanize.Bytes.Uint64(tablesTotalSize(i.Tables))),
+		redact.Safe(i.Score))
 }
 
 // CompactionInfo contains the info for a compaction event.
@@ -75,6 +79,27 @@ type CompactionInfo struct {
 	TotalDuration time.Duration
 	Done          bool
 	Err           error
+
+	SingleLevelOverlappingRatio float64
+	MultiLevelOverlappingRatio  float64
+
+	// Annotations specifies additional info to appear in a compaction's event log line
+	Annotations compactionAnnotations
+}
+
+type compactionAnnotations []string
+
+// SafeFormat implements redact.SafeFormatter.
+func (ca compactionAnnotations) SafeFormat(w redact.SafePrinter, _ rune) {
+	if len(ca) == 0 {
+		return
+	}
+	for i := range ca {
+		if i != 0 {
+			w.Print(" ")
+		}
+		w.Printf("%s", redact.SafeString(ca[i]))
+	}
 }
 
 func (i CompactionInfo) String() string {
@@ -90,12 +115,17 @@ func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 	}
 
 	if !i.Done {
-		w.Printf("[JOB %d] compacting(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
+		w.Printf("[JOB %d] compacting(%s) ",
-		w.Print(levelInfos(i.Input))
+			redact.Safe(i.JobID),
+			redact.SafeString(i.Reason))
+		w.Printf("%s", i.Annotations)
+		w.Printf("%s; ", levelInfos(i.Input))
+		w.Printf("OverlappingRatio: Single %.2f, Multi %.2f", i.SingleLevelOverlappingRatio, i.MultiLevelOverlappingRatio)
 		return
 	}
 	outputSize := tablesTotalSize(i.Output.Tables)
 	w.Printf("[JOB %d] compacted(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
+	w.Printf("%s", i.Annotations)
 	w.Print(levelInfos(i.Input))
 	w.Printf(" -> L%d [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
 		redact.Safe(i.Output.Level),
@@ -202,7 +232,7 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 			if j > 0 {
 				w.Printf(" +")
 			}
-			w.Printf(" L%d:%s (%s)", level, redact.Safe(file.FileNum), humanize.Bytes.Uint64(file.Size))
+			w.Printf(" L%d:%s (%s)", level, file.FileNum, humanize.Bytes.Uint64(file.Size))
 		}
 		w.Printf(" in %.1fs (%.1fs total), output rate %s/s",
 			redact.Safe(i.Duration.Seconds()),
@@ -231,7 +261,7 @@ func (i ManifestCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		w.Printf("[JOB %d] MANIFEST create error: %s", redact.Safe(i.JobID), i.Err)
 		return
 	}
-	w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+	w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), i.FileNum)
 }
 
 // ManifestDeleteInfo contains the info for a Manifest deletion event.
@@ -253,7 +283,7 @@ func (i ManifestDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		w.Printf("[JOB %d] MANIFEST delete error: %s", redact.Safe(i.JobID), i.Err)
 		return
 	}
-	w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+	w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), i.FileNum)
 }
 
 // TableCreateInfo contains the info for a table creation event.
@@ -273,7 +303,7 @@ func (i TableCreateInfo) String() string {
 // SafeFormat implements redact.SafeFormatter.
 func (i TableCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 	w.Printf("[JOB %d] %s: sstable created %s",
-		redact.Safe(i.JobID), redact.Safe(i.Reason), redact.Safe(i.FileNum))
+		redact.Safe(i.JobID), redact.Safe(i.Reason), i.FileNum)
 }
 
 // TableDeleteInfo contains the info for a table deletion event.
@@ -292,10 +322,10 @@ func (i TableDeleteInfo) String() string {
 func (i TableDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 	if i.Err != nil {
 		w.Printf("[JOB %d] sstable delete error %s: %s",
-			redact.Safe(i.JobID), redact.Safe(i.FileNum), i.Err)
+			redact.Safe(i.JobID), i.FileNum, i.Err)
 		return
 	}
-	w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+	w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), i.FileNum)
 }
 
 // TableIngestInfo contains the info for a table ingestion event.
@@ -341,7 +371,7 @@ func (i TableIngestInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		if !i.flushable {
 			levelStr = fmt.Sprintf("L%d:", t.Level)
 		}
-		w.Printf(" %s%s (%s)", redact.Safe(levelStr), redact.Safe(t.FileNum),
+		w.Printf(" %s%s (%s)", redact.Safe(levelStr), t.FileNum,
 			redact.Safe(humanize.Bytes.Uint64(t.Size)))
 	}
 }
@@ -403,12 +433,12 @@ func (i WALCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 	}
 
 	if i.RecycledFileNum == 0 {
-		w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+		w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), i.FileNum)
 		return
 	}
 
 	w.Printf("[JOB %d] WAL created %s (recycled %s)",
-		redact.Safe(i.JobID), redact.Safe(i.FileNum), redact.Safe(i.RecycledFileNum))
+		redact.Safe(i.JobID), i.FileNum, i.RecycledFileNum)
 }
 
 // WALDeleteInfo contains the info for a WAL deletion event.
@@ -430,7 +460,7 @@ func (i WALDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
 		w.Printf("[JOB %d] WAL delete error: %s", redact.Safe(i.JobID), i.Err)
 		return
 	}
-	w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
+	w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), i.FileNum)
 }
 
 // WriteStallBeginInfo contains the info for a write stall begin event.

vendor/github.com/cockroachdb/pebble/external_iterator.go

@@ -311,7 +311,7 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) {
 					base.InternalKeySeqNumMax,
 					it.opts.LowerBound, it.opts.UpperBound,
 					&it.hasPrefix, &it.prefixOrFullSeekKey,
-					true /* onlySets */, &it.rangeKey.internal,
+					false /* internalKeys */, &it.rangeKey.internal,
 				)
 				for i := range rangeKeyIters {
 					it.rangeKey.iterConfig.AddLevel(rangeKeyIters[i])

vendor/github.com/cockroachdb/pebble/flushable.go

@@ -117,8 +117,7 @@ type flushableList []*flushableEntry
 // ingesting sstables which are added to the flushable list.
 type ingestedFlushable struct {
 	files            []physicalMeta
-	cmp              Compare
+	comparer         *Comparer
-	split            Split
 	newIters         tableNewIters
 	newRangeKeyIters keyspan.TableNewSpanIter
 
@@ -131,8 +130,7 @@ type ingestedFlushable struct {
 
 func newIngestedFlushable(
 	files []*fileMetadata,
-	cmp Compare,
+	comparer *Comparer,
-	split Split,
 	newIters tableNewIters,
 	newRangeKeyIters keyspan.TableNewSpanIter,
 ) *ingestedFlushable {
@@ -147,12 +145,11 @@ func newIngestedFlushable(
 
 	ret := &ingestedFlushable{
 		files:            physicalFiles,
-		cmp:              cmp,
+		comparer:         comparer,
-		split:            split,
 		newIters:         newIters,
 		newRangeKeyIters: newRangeKeyIters,
 		// slice is immutable and can be set once and used many times.
-		slice:        manifest.NewLevelSliceKeySorted(cmp, files),
+		slice:        manifest.NewLevelSliceKeySorted(comparer.Compare, files),
 		hasRangeKeys: hasRangeKeys,
 	}
 
@@ -173,7 +170,7 @@ func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator {
 	// aren't truly levels in the lsm. Right now, the encoding only supports
 	// L0 sublevels, and the rest of the levels in the lsm.
 	return newLevelIter(
-		opts, s.cmp, s.split, s.newIters, s.slice.Iter(), manifest.Level(0), internalIterOpts{},
+		opts, s.comparer, s.newIters, s.slice.Iter(), manifest.Level(0), internalIterOpts{},
 	)
 }
 
@@ -206,7 +203,7 @@ func (s *ingestedFlushable) constructRangeDelIter(
 // surface range deletes is more efficient.
 func (s *ingestedFlushable) newRangeDelIter(_ *IterOptions) keyspan.FragmentIterator {
 	return keyspan.NewLevelIter(
-		keyspan.SpanIterOptions{}, s.cmp,
+		keyspan.SpanIterOptions{}, s.comparer.Compare,
 		s.constructRangeDelIter, s.slice.Iter(), manifest.Level(0),
 		manifest.KeyTypePoint,
 	)
@@ -219,7 +216,7 @@ func (s *ingestedFlushable) newRangeKeyIter(o *IterOptions) keyspan.FragmentIter
 	}
 
 	return keyspan.NewLevelIter(
-		keyspan.SpanIterOptions{}, s.cmp, s.newRangeKeyIters,
+		keyspan.SpanIterOptions{}, s.comparer.Compare, s.newRangeKeyIters,
 		s.slice.Iter(), manifest.Level(0), manifest.KeyTypeRange,
 	)
 }

vendor/github.com/cockroachdb/pebble/format_major_version.go

@@ -28,10 +28,13 @@ import (
 // FormatVersion that the default corresponds to may change with time.
 type FormatMajorVersion uint64
 
+// SafeValue implements redact.SafeValue.
+func (v FormatMajorVersion) SafeValue() {}
+
 // String implements fmt.Stringer.
 func (v FormatMajorVersion) String() string {
-	// NB: This must not change. It's used as the value for the the
+	// NB: This must not change. It's used as the value for the on-disk
-	// on-disk version marker file.
+	// version marker file.
 	//
 	// Specifically, this value must always parse as a base 10 integer
 	// that fits in a uint64. We format it as zero-padded, 3-digit
@@ -110,12 +113,12 @@ const (
 
 	// 23.1 versions.
 
-	// FormatUnusedPrePebblev1MarkedCompacted is an unused format major version.
+	// formatUnusedPrePebblev1MarkedCompacted is an unused format major version.
 	// This format major version was originally intended to ship in the 23.1
 	// release. It was later decided that this should be deferred until a
 	// subsequent release. The original ordering is preserved so as not to
 	// introduce breaking changes in Cockroach.
-	FormatUnusedPrePebblev1MarkedCompacted
+	formatUnusedPrePebblev1MarkedCompacted
 
 	// FormatSSTableValueBlocks is a format major version that adds support for
 	// storing values in value blocks in the sstable. Value block support is not
@@ -151,20 +154,20 @@ const (
 	// compactions for files marked for compaction are complete.
 	FormatPrePebblev1MarkedCompacted
 
-	// ExperimentalFormatDeleteSizedAndObsolete is a format major version that adds support
+	// FormatDeleteSizedAndObsolete is a format major version that adds support
 	// for deletion tombstones that encode the size of the value they're
 	// expected to delete. This format major version is required before the
 	// associated key kind may be committed through batch applications or
 	// ingests. It also adds support for keys that are marked obsolete (see
 	// sstable/format.go for details).
-	ExperimentalFormatDeleteSizedAndObsolete
+	FormatDeleteSizedAndObsolete
 
-	// ExperimentalFormatVirtualSSTables is a format major version that adds support for
+	// FormatVirtualSSTables is a format major version that adds support for
 	// virtual sstables that can reference a sub-range of keys in an underlying
 	// physical sstable. This information is persisted through new,
 	// backward-incompatible fields in the Manifest, and therefore requires
 	// a format major version.
-	ExperimentalFormatVirtualSSTables
+	FormatVirtualSSTables
 
 	// internalFormatNewest holds the newest format major version, including
 	// experimental ones excluded from the exported FormatNewest constant until
@@ -172,8 +175,7 @@ const (
 	internalFormatNewest FormatMajorVersion = iota - 1
 
 	// FormatNewest always contains the most recent format major version.
-	// TODO(jackson): restore FormatNewest to internalFormatNewest.
+	FormatNewest FormatMajorVersion = internalFormatNewest
-	FormatNewest FormatMajorVersion = FormatPrePebblev1MarkedCompacted
 )
 
 // MaxTableFormat returns the maximum sstable.TableFormat that can be used at
@@ -187,11 +189,11 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
 		FormatSplitUserKeysMarkedCompacted:
 		return sstable.TableFormatPebblev1
 	case FormatRangeKeys, FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
-		FormatUnusedPrePebblev1MarkedCompacted:
+		formatUnusedPrePebblev1MarkedCompacted:
 		return sstable.TableFormatPebblev2
 	case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
 		return sstable.TableFormatPebblev3
-	case ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables:
+	case FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
 		return sstable.TableFormatPebblev4
 	default:
 		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
@@ -208,15 +210,27 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
 		FormatRangeKeys:
 		return sstable.TableFormatLevelDB
 	case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked,
-		FormatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
+		formatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks,
 		FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
-		ExperimentalFormatDeleteSizedAndObsolete, ExperimentalFormatVirtualSSTables:
+		FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
 		return sstable.TableFormatPebblev1
 	default:
 		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
 	}
 }
 
+// orderingInvariants returns an enum encoding the set of invariants that must
+// hold within the receiver format major version. Invariants only get stricter
+// as the format major version advances, so it is okay to retrieve the
+// invariants from the current format major version and by the time the
+// invariants are enforced, the format major version has advanced.
+func (v FormatMajorVersion) orderingInvariants() manifest.OrderingInvariants {
+	if v < FormatSplitUserKeysMarkedCompacted {
+		return manifest.AllowSplitUserKeys
+	}
+	return manifest.ProhibitSplitUserKeys
+}
+
 // formatMajorVersionMigrations defines the migrations from one format
 // major version to the next. Each migration is defined as a closure
 // which will be invoked on the database before the new format major
@@ -324,9 +338,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 		}
 		return d.finalizeFormatVersUpgrade(FormatPrePebblev1Marked)
 	},
-	FormatUnusedPrePebblev1MarkedCompacted: func(d *DB) error {
+	formatUnusedPrePebblev1MarkedCompacted: func(d *DB) error {
 		// Intentional no-op.
-		return d.finalizeFormatVersUpgrade(FormatUnusedPrePebblev1MarkedCompacted)
+		return d.finalizeFormatVersUpgrade(formatUnusedPrePebblev1MarkedCompacted)
 	},
 	FormatSSTableValueBlocks: func(d *DB) error {
 		return d.finalizeFormatVersUpgrade(FormatSSTableValueBlocks)
@@ -343,11 +357,11 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 		}
 		return d.finalizeFormatVersUpgrade(FormatPrePebblev1MarkedCompacted)
 	},
-	ExperimentalFormatDeleteSizedAndObsolete: func(d *DB) error {
+	FormatDeleteSizedAndObsolete: func(d *DB) error {
-		return d.finalizeFormatVersUpgrade(ExperimentalFormatDeleteSizedAndObsolete)
+		return d.finalizeFormatVersUpgrade(FormatDeleteSizedAndObsolete)
 	},
-	ExperimentalFormatVirtualSSTables: func(d *DB) error {
+	FormatVirtualSSTables: func(d *DB) error {
-		return d.finalizeFormatVersUpgrade(ExperimentalFormatVirtualSSTables)
+		return d.finalizeFormatVersUpgrade(FormatVirtualSSTables)
 	},
 }
 

vendor/github.com/cockroachdb/pebble/get_iter.go

@@ -19,9 +19,7 @@ import (
 // lazily.
 type getIter struct {
 	logger       Logger
-	cmp          Compare
+	comparer     *Comparer
-	equal        Equal
-	split        Split
 	newIters     tableNewIters
 	snapshot     uint64
 	key          []byte
@@ -84,7 +82,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 			// key. Every call to levelIter.Next() potentially switches to a new
 			// table and thus reinitializes rangeDelIter.
 			if g.rangeDelIter != nil {
-				g.tombstone = keyspan.Get(g.cmp, g.rangeDelIter, g.key)
+				g.tombstone = keyspan.Get(g.comparer.Compare, g.rangeDelIter, g.key)
 				if g.err = g.rangeDelIter.Close(); g.err != nil {
 					return nil, base.LazyValue{}
 				}
@@ -102,7 +100,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 					g.iter = nil
 					return nil, base.LazyValue{}
 				}
-				if g.equal(g.key, key.UserKey) {
+				if g.comparer.Equal(g.key, key.UserKey) {
 					if !key.Visible(g.snapshot, base.InternalKeySeqNumMax) {
 						g.iterKey, g.iterValue = g.iter.Next()
 						continue
@@ -160,7 +158,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 				files := g.l0[n-1].Iter()
 				g.l0 = g.l0[:n-1]
 				iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
-				g.levelIter.init(context.Background(), iterOpts, g.cmp, g.split, g.newIters,
+				g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters,
 					files, manifest.L0Sublevel(n), internalIterOpts{})
 				g.levelIter.initRangeDel(&g.rangeDelIter)
 				bc := levelIterBoundaryContext{}
@@ -170,8 +168,8 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 				// Compute the key prefix for bloom filtering if split function is
 				// specified, or use the user key as default.
 				prefix := g.key
-				if g.split != nil {
+				if g.comparer.Split != nil {
-					prefix = g.key[:g.split(g.key)]
+					prefix = g.key[:g.comparer.Split(g.key)]
 				}
 				g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
 				if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {
@@ -192,7 +190,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 		}
 
 		iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot}
-		g.levelIter.init(context.Background(), iterOpts, g.cmp, g.split, g.newIters,
+		g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters,
 			g.version.Levels[g.level].Iter(), manifest.Level(g.level), internalIterOpts{})
 		g.levelIter.initRangeDel(&g.rangeDelIter)
 		bc := levelIterBoundaryContext{}
@@ -203,8 +201,8 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) {
 		// Compute the key prefix for bloom filtering if split function is
 		// specified, or use the user key as default.
 		prefix := g.key
-		if g.split != nil {
+		if g.comparer.Split != nil {
-			prefix = g.key[:g.split(g.key)]
+			prefix = g.key[:g.comparer.Split(g.key)]
 		}
 		g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone)
 		if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey {

vendor/github.com/cockroachdb/pebble/ingest.go

@@ -113,11 +113,28 @@ func ingestSynthesizeShared(
 	meta.FileBacking.Size = sm.Size
 	if sm.LargestRangeKey.Valid() && sm.LargestRangeKey.UserKey != nil {
 		// Initialize meta.{HasRangeKeys,Smallest,Largest}, etc.
-		meta.ExtendRangeKeyBounds(opts.Comparer.Compare, sm.SmallestRangeKey, sm.LargestRangeKey)
+		//
+		// NB: We create new internal keys and pass them into ExternalRangeKeyBounds
+		// so that we can sub a zero sequence number into the bounds. We can set
+		// the sequence number to anything here; it'll be reset in ingestUpdateSeqNum
+		// anyway. However we do need to use the same sequence number across all
+		// bound keys at this step so that we end up with bounds that are consistent
+		// across point/range keys.
+		smallestRangeKey := base.MakeInternalKey(sm.SmallestRangeKey.UserKey, 0, sm.SmallestRangeKey.Kind())
+		largestRangeKey := base.MakeExclusiveSentinelKey(sm.LargestRangeKey.Kind(), sm.LargestRangeKey.UserKey)
+		meta.ExtendRangeKeyBounds(opts.Comparer.Compare, smallestRangeKey, largestRangeKey)
 	}
 	if sm.LargestPointKey.Valid() && sm.LargestPointKey.UserKey != nil {
 		// Initialize meta.{HasPointKeys,Smallest,Largest}, etc.
-		meta.ExtendPointKeyBounds(opts.Comparer.Compare, sm.SmallestPointKey, sm.LargestPointKey)
+		//
+		// See point above in the ExtendRangeKeyBounds call on why we use a zero
+		// sequence number here.
+		smallestPointKey := base.MakeInternalKey(sm.SmallestPointKey.UserKey, 0, sm.SmallestPointKey.Kind())
+		largestPointKey := base.MakeInternalKey(sm.LargestPointKey.UserKey, 0, sm.LargestPointKey.Kind())
+		if sm.LargestPointKey.IsExclusiveSentinel() {
+			largestPointKey = base.MakeRangeDeleteSentinelKey(sm.LargestPointKey.UserKey)
+		}
+		meta.ExtendPointKeyBounds(opts.Comparer.Compare, smallestPointKey, largestPointKey)
 	}
 	if err := meta.Validate(opts.Comparer.Compare, opts.Comparer.FormatKey); err != nil {
 		return nil, err
@@ -780,16 +797,21 @@ func overlapWithIterator(
 	return computeOverlapWithSpans(*rangeDelIter)
 }
 
+// ingestTargetLevel returns the target level for a file being ingested.
+// If suggestSplit is true, it accounts for ingest-time splitting as part of
+// its target level calculation, and if a split candidate is found, that file
+// is returned as the splitFile.
 func ingestTargetLevel(
 	newIters tableNewIters,
 	newRangeKeyIter keyspan.TableNewSpanIter,
 	iterOps IterOptions,
-	cmp Compare,
+	comparer *Comparer,
 	v *version,
 	baseLevel int,
 	compactions map[*compaction]struct{},
 	meta *fileMetadata,
-) (int, error) {
+	suggestSplit bool,
+) (targetLevel int, splitFile *fileMetadata, err error) {
 	// Find the lowest level which does not have any files which overlap meta. We
 	// search from L0 to L6 looking for whether there are any files in the level
 	// which overlap meta. We want the "lowest" level (where lower means
@@ -804,6 +826,14 @@ func ingestTargetLevel(
 	//   violate the sequence number invariant.
 	// - no file boundary overlap with level i, since that will violate the
 	//   invariant that files do not overlap in levels i > 0.
+	//   - if there is only a file overlap at a given level, and no data overlap,
+	//     we can still slot a file at that level. We return the fileMetadata with
+	//     which we have file boundary overlap (must be only one file, as sstable
+	//     bounds are usually tight on user keys) and the caller is expected to split
+	//     that sstable into two virtual sstables, allowing this file to go into that
+	//     level. Note that if we have file boundary overlap with two files, which
+	//     should only happen on rare occasions, we treat it as data overlap and
+	//     don't use this optimization.
 	//
 	// The file boundary overlap check is simpler to conceptualize. Consider the
 	// following example, in which the ingested file lies completely before or
@@ -848,16 +878,14 @@ func ingestTargetLevel(
 	// existing point that falls within the ingested table bounds as being "data
 	// overlap".
 
-	targetLevel := 0
-
 	// This assertion implicitly checks that we have the current version of
 	// the metadata.
 	if v.L0Sublevels == nil {
-		return 0, errors.AssertionFailedf("could not read L0 sublevels")
+		return 0, nil, errors.AssertionFailedf("could not read L0 sublevels")
 	}
 	// Check for overlap over the keys of L0 by iterating over the sublevels.
 	for subLevel := 0; subLevel < len(v.L0SublevelFiles); subLevel++ {
-		iter := newLevelIter(iterOps, cmp, nil /* split */, newIters,
+		iter := newLevelIter(iterOps, comparer, newIters,
 			v.L0Sublevels.Levels[subLevel].Iter(), manifest.Level(0), internalIterOpts{})
 
 		var rangeDelIter keyspan.FragmentIterator
@@ -867,7 +895,7 @@ func ingestTargetLevel(
 
 		levelIter := keyspan.LevelIter{}
 		levelIter.Init(
-			keyspan.SpanIterOptions{}, cmp, newRangeKeyIter,
+			keyspan.SpanIterOptions{}, comparer.Compare, newRangeKeyIter,
 			v.L0Sublevels.Levels[subLevel].Iter(), manifest.Level(0), manifest.KeyTypeRange,
 		)
 
@@ -875,20 +903,20 @@ func ingestTargetLevel(
 			smallest: meta.Smallest,
 			largest:  meta.Largest,
 		}
-		overlap := overlapWithIterator(iter, &rangeDelIter, &levelIter, kr, cmp)
+		overlap := overlapWithIterator(iter, &rangeDelIter, &levelIter, kr, comparer.Compare)
 		err := iter.Close() // Closes range del iter as well.
 		err = firstError(err, levelIter.Close())
 		if err != nil {
-			return 0, err
+			return 0, nil, err
 		}
 		if overlap {
-			return targetLevel, nil
+			return targetLevel, nil, nil
 		}
 	}
 
 	level := baseLevel
 	for ; level < numLevels; level++ {
-		levelIter := newLevelIter(iterOps, cmp, nil /* split */, newIters,
+		levelIter := newLevelIter(iterOps, comparer, newIters,
 			v.Levels[level].Iter(), manifest.Level(level), internalIterOpts{})
 		var rangeDelIter keyspan.FragmentIterator
 		// Pass in a non-nil pointer to rangeDelIter so that levelIter.findFileGE
@@ -897,7 +925,7 @@ func ingestTargetLevel(
 
 		rkeyLevelIter := &keyspan.LevelIter{}
 		rkeyLevelIter.Init(
-			keyspan.SpanIterOptions{}, cmp, newRangeKeyIter,
+			keyspan.SpanIterOptions{}, comparer.Compare, newRangeKeyIter,
 			v.Levels[level].Iter(), manifest.Level(level), manifest.KeyTypeRange,
 		)
 
@@ -905,46 +933,68 @@ func ingestTargetLevel(
 			smallest: meta.Smallest,
 			largest:  meta.Largest,
 		}
-		overlap := overlapWithIterator(levelIter, &rangeDelIter, rkeyLevelIter, kr, cmp)
+		overlap := overlapWithIterator(levelIter, &rangeDelIter, rkeyLevelIter, kr, comparer.Compare)
 		err := levelIter.Close() // Closes range del iter as well.
 		err = firstError(err, rkeyLevelIter.Close())
 		if err != nil {
-			return 0, err
+			return 0, nil, err
 		}
 		if overlap {
-			return targetLevel, nil
+			return targetLevel, splitFile, nil
 		}
 
 		// Check boundary overlap.
-		boundaryOverlaps := v.Overlaps(level, cmp, meta.Smallest.UserKey,
+		var candidateSplitFile *fileMetadata
+		boundaryOverlaps := v.Overlaps(level, comparer.Compare, meta.Smallest.UserKey,
 			meta.Largest.UserKey, meta.Largest.IsExclusiveSentinel())
 		if !boundaryOverlaps.Empty() {
-			continue
+			// We are already guaranteed to not have any data overlaps with files
+			// in boundaryOverlaps, otherwise we'd have returned in the above if
+			// statements. Use this, plus boundaryOverlaps.Len() == 1 to detect for
+			// the case where we can slot this file into the current level despite
+			// a boundary overlap, by splitting one existing file into two virtual
+			// sstables.
+			if suggestSplit && boundaryOverlaps.Len() == 1 {
+				iter := boundaryOverlaps.Iter()
+				candidateSplitFile = iter.First()
+			} else {
+				// We either don't want to suggest ingest-time splits (i.e.
+				// !suggestSplit), or we boundary-overlapped with more than one file.
+				continue
+			}
 		}
 
-		// Check boundary overlap with any ongoing compactions.
+		// Check boundary overlap with any ongoing compactions. We consider an
+		// overlapping compaction that's writing files to an output level as
+		// equivalent to boundary overlap with files in that output level.
 		//
-		// We cannot check for data overlap with the new SSTs compaction will
+		// We cannot check for data overlap with the new SSTs compaction will produce
-		// produce since compaction hasn't been done yet. However, there's no need
+		// since compaction hasn't been done yet. However, there's no need to check
-		// to check since all keys in them will either be from c.startLevel or
+		// since all keys in them will be from levels in [c.startLevel,
-		// c.outputLevel, both levels having their data overlap already tested
+		// c.outputLevel], and all those levels have already had their data overlap
-		// negative (else we'd have returned earlier).
+		// tested negative (else we'd have returned earlier).
+		//
+		// An alternative approach would be to cancel these compactions and proceed
+		// with an ingest-time split on this level if necessary. However, compaction
+		// cancellation can result in significant wasted effort and is best avoided
+		// unless necessary.
 		overlaps := false
 		for c := range compactions {
 			if c.outputLevel == nil || level != c.outputLevel.level {
 				continue
 			}
-			if cmp(meta.Smallest.UserKey, c.largest.UserKey) <= 0 &&
+			if comparer.Compare(meta.Smallest.UserKey, c.largest.UserKey) <= 0 &&
-				cmp(meta.Largest.UserKey, c.smallest.UserKey) >= 0 {
+				comparer.Compare(meta.Largest.UserKey, c.smallest.UserKey) >= 0 {
 				overlaps = true
 				break
 			}
 		}
 		if !overlaps {
 			targetLevel = level
+			splitFile = candidateSplitFile
 		}
 	}
-	return targetLevel, nil
+	return targetLevel, splitFile, nil
 }
 
 // Ingest ingests a set of sstables into the DB. Ingestion of the files is
@@ -1126,7 +1176,7 @@ func (d *DB) newIngestedFlushableEntry(
 		return nil, err
 	}
 
-	f := newIngestedFlushable(meta, d.cmp, d.split, d.newIters, d.tableNewRangeKeyIter)
+	f := newIngestedFlushable(meta, d.opts.Comparer, d.newIters, d.tableNewRangeKeyIter)
 
 	// NB: The logNum/seqNum are the WAL number which we're writing this entry
 	// to and the sequence number within the WAL which we'll write this entry
@@ -1226,7 +1276,7 @@ func (d *DB) ingest(
 	if len(shared) > 0 && d.opts.Experimental.RemoteStorage == nil {
 		panic("cannot ingest shared sstables with nil SharedStorage")
 	}
-	if (exciseSpan.Valid() || len(shared) > 0 || len(external) > 0) && d.FormatMajorVersion() < ExperimentalFormatVirtualSSTables {
+	if (exciseSpan.Valid() || len(shared) > 0 || len(external) > 0) && d.FormatMajorVersion() < FormatVirtualSSTables {
 		return IngestOperationStats{}, errors.New("pebble: format major version too old for excise, shared or external sstable ingestion")
 	}
 	// Allocate file numbers for all of the files being ingested and mark them as
@@ -1558,7 +1608,7 @@ func (d *DB) excise(
 	var iter internalIterator
 	var rangeDelIter keyspan.FragmentIterator
 	var rangeKeyIter keyspan.FragmentIterator
-	backingTableCreated := false
+	needsBacking := false
 	// Create a file to the left of the excise span, if necessary.
 	// The bounds of this file will be [m.Smallest, lastKeyBefore(exciseSpan.Start)].
 	//
@@ -1672,14 +1722,20 @@ func (d *DB) excise(
 			leftFile.ValidateVirtual(m)
 			d.checkVirtualBounds(leftFile)
 			ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: level, Meta: leftFile})
-			ve.CreatedBackingTables = append(ve.CreatedBackingTables, leftFile.FileBacking)
+			needsBacking = true
-			backingTableCreated = true
 			numCreatedFiles++
 		}
 	}
 	// Create a file to the right, if necessary.
 	if exciseSpan.Contains(d.cmp, m.Largest) {
 		// No key exists to the right of the excise span in this file.
+		if needsBacking && !m.Virtual {
+			// If m is virtual, then its file backing is already known to the manifest.
+			// We don't need to create another file backing. Note that there must be
+			// only one CreatedBackingTables entry per backing sstable. This is
+			// indicated by the VersionEdit.CreatedBackingTables invariant.
+			ve.CreatedBackingTables = append(ve.CreatedBackingTables, m.FileBacking)
+		}
 		return ve.NewFiles[len(ve.NewFiles)-numCreatedFiles:], nil
 	}
 	// Create a new file, rightFile, between [firstKeyAfter(exciseSpan.End), m.Largest].
@@ -1734,7 +1790,8 @@ func (d *DB) excise(
 			smallestPointKey.UserKey = firstRangeDel
 			rightFile.ExtendPointKeyBounds(d.cmp, smallestPointKey, largestPointKey)
 		}
-	} else if m.HasRangeKeys && !exciseSpan.Contains(d.cmp, m.LargestRangeKey) {
+	}
+	if m.HasRangeKeys && !exciseSpan.Contains(d.cmp, m.LargestRangeKey) {
 		// This file will contain range keys.
 		largestRangeKey := m.LargestRangeKey
 		if rangeKeyIter == nil {
@@ -1781,13 +1838,18 @@ func (d *DB) excise(
 		rightFile.ValidateVirtual(m)
 		d.checkVirtualBounds(rightFile)
 		ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: level, Meta: rightFile})
-		if !backingTableCreated {
+		needsBacking = true
-			ve.CreatedBackingTables = append(ve.CreatedBackingTables, rightFile.FileBacking)
-			backingTableCreated = true
-		}
 		numCreatedFiles++
 	}
 
+	if needsBacking && !m.Virtual {
+		// If m is virtual, then its file backing is already known to the manifest.
+		// We don't need to create another file backing. Note that there must be
+		// only one CreatedBackingTables entry per backing sstable. This is
+		// indicated by the VersionEdit.CreatedBackingTables invariant.
+		ve.CreatedBackingTables = append(ve.CreatedBackingTables, m.FileBacking)
+	}
+
 	if err := rightFile.Validate(d.cmp, d.opts.Comparer.FormatKey); err != nil {
 		return nil, err
 	}
@@ -1798,12 +1860,129 @@ type ingestTargetLevelFunc func(
 	newIters tableNewIters,
 	newRangeKeyIter keyspan.TableNewSpanIter,
 	iterOps IterOptions,
-	cmp Compare,
+	comparer *Comparer,
 	v *version,
 	baseLevel int,
 	compactions map[*compaction]struct{},
 	meta *fileMetadata,
-) (int, error)
+	suggestSplit bool,
+) (int, *fileMetadata, error)
+
+type ingestSplitFile struct {
+	// ingestFile is the file being ingested.
+	ingestFile *fileMetadata
+	// splitFile is the file that needs to be split to allow ingestFile to slot
+	// into `level` level.
+	splitFile *fileMetadata
+	// The level where ingestFile will go (and where splitFile already is).
+	level int
+}
+
+// ingestSplit splits files specified in `files` and updates ve in-place to
+// account for existing files getting split into two virtual sstables. The map
+// `replacedFiles` contains an in-progress map of all files that have been
+// replaced with new virtual sstables in this version edit so far, which is also
+// updated in-place.
+//
+// d.mu as well as the manifest lock must be held when calling this method.
+func (d *DB) ingestSplit(
+	ve *versionEdit,
+	updateMetrics func(*fileMetadata, int, []newFileEntry),
+	files []ingestSplitFile,
+	replacedFiles map[base.FileNum][]newFileEntry,
+) error {
+	for _, s := range files {
+		// replacedFiles can be thought of as a tree, where we start iterating with
+		// s.splitFile and run its fileNum through replacedFiles, then find which of
+		// the replaced files overlaps with s.ingestFile, which becomes the new
+		// splitFile, then we check splitFile's replacements in replacedFiles again
+		// for overlap with s.ingestFile, and so on until we either can't find the
+		// current splitFile in replacedFiles (i.e. that's the file that now needs to
+		// be split), or we don't find a file that overlaps with s.ingestFile, which
+		// means a prior ingest split already produced enough room for s.ingestFile
+		// to go into this level without necessitating another ingest split.
+		splitFile := s.splitFile
+		for splitFile != nil {
+			replaced, ok := replacedFiles[splitFile.FileNum]
+			if !ok {
+				break
+			}
+			updatedSplitFile := false
+			for i := range replaced {
+				if replaced[i].Meta.Overlaps(d.cmp, s.ingestFile.Smallest.UserKey, s.ingestFile.Largest.UserKey, s.ingestFile.Largest.IsExclusiveSentinel()) {
+					if updatedSplitFile {
+						// This should never happen because the earlier ingestTargetLevel
+						// function only finds split file candidates that are guaranteed to
+						// have no data overlap, only boundary overlap. See the comments
+						// in that method to see the definitions of data vs boundary
+						// overlap. That, plus the fact that files in `replaced` are
+						// guaranteed to have file bounds that are tight on user keys
+						// (as that's what `d.excise` produces), means that the only case
+						// where we overlap with two or more files in `replaced` is if we
+						// actually had data overlap all along, or if the ingestion files
+						// were overlapping, either of which is an invariant violation.
+						panic("updated with two files in ingestSplit")
+					}
+					splitFile = replaced[i].Meta
+					updatedSplitFile = true
+				}
+			}
+			if !updatedSplitFile {
+				// None of the replaced files overlapped with the file being ingested.
+				// This can happen if we've already excised a span overlapping with
+				// this file, or if we have consecutive ingested files that can slide
+				// within the same gap between keys in an existing file. For instance,
+				// if an existing file has keys a and g and we're ingesting b-c, d-e,
+				// the first loop iteration will split the existing file into one that
+				// ends in a and another that starts at g, and the second iteration will
+				// fall into this case and require no splitting.
+				//
+				// No splitting necessary.
+				splitFile = nil
+			}
+		}
+		if splitFile == nil {
+			continue
+		}
+		// NB: excise operates on [start, end). We're splitting at [start, end]
+		// (assuming !s.ingestFile.Largest.IsExclusiveSentinel()). The conflation
+		// of exclusive vs inclusive end bounds should not make a difference here
+		// as we're guaranteed to not have any data overlap between splitFile and
+		// s.ingestFile, so panic if we do see a newly added file with an endKey
+		// equalling s.ingestFile.Largest, and !s.ingestFile.Largest.IsExclusiveSentinel()
+		added, err := d.excise(KeyRange{Start: s.ingestFile.Smallest.UserKey, End: s.ingestFile.Largest.UserKey}, splitFile, ve, s.level)
+		if err != nil {
+			return err
+		}
+		if _, ok := ve.DeletedFiles[deletedFileEntry{
+			Level:   s.level,
+			FileNum: splitFile.FileNum,
+		}]; !ok {
+			panic("did not split file that was expected to be split")
+		}
+		replacedFiles[splitFile.FileNum] = added
+		for i := range added {
+			if s.ingestFile.Overlaps(d.cmp, added[i].Meta.Smallest.UserKey, added[i].Meta.Largest.UserKey, added[i].Meta.Largest.IsExclusiveSentinel()) {
+				panic("ingest-time split produced a file that overlaps with ingested file")
+			}
+		}
+		updateMetrics(splitFile, s.level, added)
+	}
+	// Flatten the version edit by removing any entries from ve.NewFiles that
+	// are also in ve.DeletedFiles.
+	newNewFiles := ve.NewFiles[:0]
+	for i := range ve.NewFiles {
+		fn := ve.NewFiles[i].Meta.FileNum
+		deEntry := deletedFileEntry{Level: ve.NewFiles[i].Level, FileNum: fn}
+		if _, ok := ve.DeletedFiles[deEntry]; ok {
+			delete(ve.DeletedFiles, deEntry)
+		} else {
+			newNewFiles = append(newNewFiles, ve.NewFiles[i])
+		}
+	}
+	ve.NewFiles = newNewFiles
+	return nil
+}
 
 func (d *DB) ingestApply(
 	jobID int,
@@ -1818,7 +1997,7 @@ func (d *DB) ingestApply(
 	ve := &versionEdit{
 		NewFiles: make([]newFileEntry, lr.fileCount),
 	}
-	if exciseSpan.Valid() {
+	if exciseSpan.Valid() || (d.opts.Experimental.IngestSplit != nil && d.opts.Experimental.IngestSplit()) {
 		ve.DeletedFiles = map[manifest.DeletedFileEntry]*manifest.FileMetadata{}
 	}
 	metrics := make(map[int]*LevelMetrics)
@@ -1843,9 +2022,17 @@ func (d *DB) ingestApply(
 		}
 	}
 
+	shouldIngestSplit := d.opts.Experimental.IngestSplit != nil &&
+		d.opts.Experimental.IngestSplit() && d.FormatMajorVersion() >= FormatVirtualSSTables
 	current := d.mu.versions.currentVersion()
 	baseLevel := d.mu.versions.picker.getBaseLevel()
 	iterOps := IterOptions{logger: d.opts.Logger}
+	// filesToSplit is a list where each element is a pair consisting of a file
+	// being ingested and a file being split to make room for an ingestion into
+	// that level. Each ingested file will appear at most once in this list. It
+	// is possible for split files to appear twice in this list.
+	filesToSplit := make([]ingestSplitFile, 0)
+	checkCompactions := false
 	for i := 0; i < lr.fileCount; i++ {
 		// Determine the lowest level in the LSM for which the sstable doesn't
 		// overlap any existing files in the level.
@@ -1878,6 +2065,7 @@ func (d *DB) ingestApply(
 			if externalFile {
 				ve.CreatedBackingTables = append(ve.CreatedBackingTables, m.FileBacking)
 			}
+			var splitFile *fileMetadata
 			if exciseSpan.Valid() && exciseSpan.Contains(d.cmp, m.Smallest) && exciseSpan.Contains(d.cmp, m.Largest) {
 				// This file fits perfectly within the excise span. We can slot it at
 				// L6, or sharedLevelsStart - 1 if we have shared files.
@@ -1890,7 +2078,31 @@ func (d *DB) ingestApply(
 					f.Level = 6
 				}
 			} else {
-				f.Level, err = findTargetLevel(d.newIters, d.tableNewRangeKeyIter, iterOps, d.cmp, current, baseLevel, d.mu.compact.inProgress, m)
+				// TODO(bilal): findTargetLevel does disk IO (reading files for data
+				// overlap) even though we're holding onto d.mu. Consider unlocking
+				// d.mu while we do this. We already hold versions.logLock so we should
+				// not see any version applications while we're at this. The one
+				// complication here would be pulling out the mu.compact.inProgress
+				// check from findTargetLevel, as that requires d.mu to be held.
+				f.Level, splitFile, err = findTargetLevel(
+					d.newIters, d.tableNewRangeKeyIter, iterOps, d.opts.Comparer, current, baseLevel, d.mu.compact.inProgress, m, shouldIngestSplit)
+			}
+
+			if splitFile != nil {
+				if invariants.Enabled {
+					if lf := current.Levels[f.Level].Find(d.cmp, splitFile); lf == nil {
+						panic("splitFile returned is not in level it should be")
+					}
+				}
+				// We take advantage of the fact that we won't drop the db mutex
+				// between now and the call to logAndApply. So, no files should
+				// get added to a new in-progress compaction at this point. We can
+				// avoid having to iterate on in-progress compactions to cancel them
+				// if none of the files being split have a compacting state.
+				if splitFile.IsCompacting() {
+					checkCompactions = true
+				}
+				filesToSplit = append(filesToSplit, ingestSplitFile{ingestFile: m, splitFile: splitFile, level: f.Level})
 			}
 		}
 		if err != nil {
@@ -1908,6 +2120,26 @@ func (d *DB) ingestApply(
 		levelMetrics.BytesIngested += m.Size
 		levelMetrics.TablesIngested++
 	}
+	// replacedFiles maps files excised due to exciseSpan (or splitFiles returned
+	// by ingestTargetLevel), to files that were created to replace it. This map
+	// is used to resolve references to split files in filesToSplit, as it is
+	// possible for a file that we want to split to no longer exist or have a
+	// newer fileMetadata due to a split induced by another ingestion file, or an
+	// excise.
+	replacedFiles := make(map[base.FileNum][]newFileEntry)
+	updateLevelMetricsOnExcise := func(m *fileMetadata, level int, added []newFileEntry) {
+		levelMetrics := metrics[level]
+		if levelMetrics == nil {
+			levelMetrics = &LevelMetrics{}
+			metrics[level] = levelMetrics
+		}
+		levelMetrics.NumFiles--
+		levelMetrics.Size -= int64(m.Size)
+		for i := range added {
+			levelMetrics.NumFiles++
+			levelMetrics.Size += int64(added[i].Meta.Size)
+		}
+	}
 	if exciseSpan.Valid() {
 		// Iterate through all levels and find files that intersect with exciseSpan.
 		//
@@ -1929,7 +2161,7 @@ func (d *DB) ingestApply(
 			iter := overlaps.Iter()
 
 			for m := iter.First(); m != nil; m = iter.Next() {
-				excised, err := d.excise(exciseSpan, m, ve, level)
+				newFiles, err := d.excise(exciseSpan, m, ve, level)
 				if err != nil {
 					return nil, err
 				}
@@ -1941,19 +2173,19 @@ func (d *DB) ingestApply(
 					// We did not excise this file.
 					continue
 				}
-				levelMetrics := metrics[level]
+				replacedFiles[m.FileNum] = newFiles
-				if levelMetrics == nil {
+				updateLevelMetricsOnExcise(m, level, newFiles)
-					levelMetrics = &LevelMetrics{}
-					metrics[level] = levelMetrics
-				}
-				levelMetrics.NumFiles--
-				levelMetrics.Size -= int64(m.Size)
-				for i := range excised {
-					levelMetrics.NumFiles++
-					levelMetrics.Size += int64(excised[i].Meta.Size)
-				}
 			}
 		}
+	}
+	if len(filesToSplit) > 0 {
+		// For the same reasons as the above call to excise, we hold the db mutex
+		// while calling this method.
+		if err := d.ingestSplit(ve, updateLevelMetricsOnExcise, filesToSplit, replacedFiles); err != nil {
+			return nil, err
+		}
+	}
+	if len(filesToSplit) > 0 || exciseSpan.Valid() {
 		for c := range d.mu.compact.inProgress {
 			if c.versionEditApplied {
 				continue
@@ -1968,22 +2200,41 @@ func (d *DB) ingestApply(
 			if exciseSpan.OverlapsInternalKeyRange(d.cmp, c.smallest, c.largest) {
 				c.cancel.Store(true)
 			}
+			// Check if this compaction's inputs have been replaced due to an
+			// ingest-time split. In that case, cancel the compaction as a newly picked
+			// compaction would need to include any new files that slid in between
+			// previously-existing files. Note that we cancel any compaction that has a
+			// file that was ingest-split as an input, even if it started before this
+			// ingestion.
+			if checkCompactions {
+				for i := range c.inputs {
+					iter := c.inputs[i].files.Iter()
+					for f := iter.First(); f != nil; f = iter.Next() {
+						if _, ok := replacedFiles[f.FileNum]; ok {
+							c.cancel.Store(true)
+							break
+						}
+					}
+				}
+			}
 		}
 		// Check for any EventuallyFileOnlySnapshots that could be watching for
 		// an excise on this span.
-		for s := d.mu.snapshots.root.next; s != &d.mu.snapshots.root; s = s.next {
+		if exciseSpan.Valid() {
-			if s.efos == nil {
+			for s := d.mu.snapshots.root.next; s != &d.mu.snapshots.root; s = s.next {
-				continue
+				if s.efos == nil {
-			}
+					continue
-			efos := s.efos
+				}
-			// TODO(bilal): We can make this faster by taking advantage of the sorted
+				efos := s.efos
-			// nature of protectedRanges to do a sort.Search, or even maintaining a
+				// TODO(bilal): We can make this faster by taking advantage of the sorted
-			// global list of all protected ranges instead of having to peer into every
+				// nature of protectedRanges to do a sort.Search, or even maintaining a
-			// snapshot.
+				// global list of all protected ranges instead of having to peer into every
-			for i := range efos.protectedRanges {
+				// snapshot.
-				if efos.protectedRanges[i].OverlapsKeyRange(d.cmp, exciseSpan) {
+				for i := range efos.protectedRanges {
-					efos.excised.Store(true)
+					if efos.protectedRanges[i].OverlapsKeyRange(d.cmp, exciseSpan) {
-					break
+						efos.excised.Store(true)
+						break
+					}
 				}
 			}
 		}
@@ -2004,28 +2255,31 @@ func (d *DB) ingestApply(
 	// The ingestion may have pushed a level over the threshold for compaction,
 	// so check to see if one is necessary and schedule it.
 	d.maybeScheduleCompaction()
-	d.maybeValidateSSTablesLocked(ve.NewFiles)
+	var toValidate []manifest.NewFileEntry
+	dedup := make(map[base.DiskFileNum]struct{})
+	for _, entry := range ve.NewFiles {
+		if _, ok := dedup[entry.Meta.FileBacking.DiskFileNum]; !ok {
+			toValidate = append(toValidate, entry)
+			dedup[entry.Meta.FileBacking.DiskFileNum] = struct{}{}
+		}
+	}
+	d.maybeValidateSSTablesLocked(toValidate)
 	return ve, nil
 }
 
 // maybeValidateSSTablesLocked adds the slice of newFileEntrys to the pending
 // queue of files to be validated, when the feature is enabled.
-// DB.mu must be locked when calling.
 //
-// TODO(bananabrick): Make sure that the ingestion step only passes in the
+// Note that if two entries with the same backing file are added twice, then the
-// physical sstables for validation here.
+// block checksums for the backing file will be validated twice.
+//
+// DB.mu must be locked when calling.
 func (d *DB) maybeValidateSSTablesLocked(newFiles []newFileEntry) {
 	// Only add to the validation queue when the feature is enabled.
 	if !d.opts.Experimental.ValidateOnIngest {
 		return
 	}
 
-	for _, f := range newFiles {
-		if f.Meta.Virtual {
-			panic("pebble: invalid call to maybeValidateSSTablesLocked")
-		}
-	}
-
 	d.mu.tableValidation.pending = append(d.mu.tableValidation.pending, newFiles...)
 	if d.shouldValidateSSTablesLocked() {
 		go d.validateSSTables()
@@ -2085,10 +2339,19 @@ func (d *DB) validateSSTables() {
 			}
 		}
 
-		err := d.tableCache.withReader(
+		var err error
-			f.Meta.PhysicalMeta(), func(r *sstable.Reader) error {
+		if f.Meta.Virtual {
-				return r.ValidateBlockChecksums()
+			err = d.tableCache.withVirtualReader(
-			})
+				f.Meta.VirtualMeta(), func(v sstable.VirtualReader) error {
+					return v.ValidateBlockChecksumsOnBacking()
+				})
+		} else {
+			err = d.tableCache.withReader(
+				f.Meta.PhysicalMeta(), func(r *sstable.Reader) error {
+					return r.ValidateBlockChecksums()
+				})
+		}
+
 		if err != nil {
 			// TODO(travers): Hook into the corruption reporting pipeline, once
 			// available. See pebble#1192.

vendor/github.com/cockroachdb/pebble/internal/base/filenames.go

@@ -20,6 +20,11 @@ type FileNum uint64
 // String returns a string representation of the file number.
 func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
 
+// SafeFormat implements redact.SafeFormatter.
+func (fn FileNum) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Printf("%06d", redact.SafeUint(fn))
+}
+
 // DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
 // called if the caller can ensure that the FileNum belongs to a physical file
 // on disk. These could be manifests, log files, physical sstables on disk, the
@@ -40,6 +45,11 @@ type DiskFileNum struct {
 
 func (dfn DiskFileNum) String() string { return dfn.fn.String() }
 
+// SafeFormat implements redact.SafeFormatter.
+func (dfn DiskFileNum) SafeFormat(w redact.SafePrinter, verb rune) {
+	dfn.fn.SafeFormat(w, verb)
+}
+
 // FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
 func (dfn DiskFileNum) FileNum() FileNum {
 	return dfn.fn

vendor/github.com/cockroachdb/pebble/internal/base/internal.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+
+	"github.com/cockroachdb/redact"
 )
 
 const (
@@ -161,6 +163,11 @@ func (k InternalKeyKind) String() string {
 	return fmt.Sprintf("UNKNOWN:%d", k)
 }
 
+// SafeFormat implements redact.SafeFormatter.
+func (k InternalKeyKind) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Print(redact.SafeString(k.String()))
+}
+
 // InternalKey is a key used for the in-memory and on-disk partial DBs that
 // make up a pebble DB.
 //
@@ -374,6 +381,11 @@ func (k InternalKey) SeqNum() uint64 {
 	return k.Trailer >> 8
 }
 
+// SeqNumFromTrailer returns the sequence number component of a trailer.
+func SeqNumFromTrailer(t uint64) uint64 {
+	return t >> 8
+}
+
 // Visible returns true if the key is visible at the specified snapshot
 // sequence number.
 func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool {

vendor/github.com/cockroachdb/pebble/internal/invalidating/iter.go

@@ -0,0 +1,162 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package invalidating
+
+import (
+	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/fastrand"
+	"github.com/cockroachdb/pebble/internal/invariants"
+)
+
+// MaybeWrapIfInvariants wraps some iterators with an invalidating iterator.
+// MaybeWrapIfInvariants does nothing in non-invariant builds.
+func MaybeWrapIfInvariants(iter base.InternalIterator) base.InternalIterator {
+	if invariants.Enabled {
+		if fastrand.Uint32n(10) == 1 {
+			return NewIter(iter)
+		}
+	}
+	return iter
+}
+
+// iter tests unsafe key/value slice reuse by modifying the last
+// returned key/value to all 1s.
+type iter struct {
+	iter        base.InternalIterator
+	lastKey     *base.InternalKey
+	lastValue   base.LazyValue
+	ignoreKinds [base.InternalKeyKindMax + 1]bool
+	err         error
+}
+
+// Option configures the behavior of an invalidating iterator.
+type Option interface {
+	apply(*iter)
+}
+
+type funcOpt func(*iter)
+
+func (f funcOpt) apply(i *iter) { f(i) }
+
+// IgnoreKinds constructs an Option that configures an invalidating iterator to
+// skip trashing k/v pairs with the provided key kinds. Some iterators provided
+// key stability guarantees for specific key kinds.
+func IgnoreKinds(kinds ...base.InternalKeyKind) Option {
+	return funcOpt(func(i *iter) {
+		for _, kind := range kinds {
+			i.ignoreKinds[kind] = true
+		}
+	})
+}
+
+// NewIter constructs a new invalidating iterator that wraps the provided
+// iterator, trashing buffers for previously returned keys.
+func NewIter(originalIterator base.InternalIterator, opts ...Option) base.InternalIterator {
+	i := &iter{iter: originalIterator}
+	for _, opt := range opts {
+		opt.apply(i)
+	}
+	return i
+}
+
+func (i *iter) update(
+	key *base.InternalKey, value base.LazyValue,
+) (*base.InternalKey, base.LazyValue) {
+	i.trashLastKV()
+	if key == nil {
+		i.lastKey = nil
+		i.lastValue = base.LazyValue{}
+		return nil, base.LazyValue{}
+	}
+
+	i.lastKey = &base.InternalKey{}
+	*i.lastKey = key.Clone()
+	i.lastValue = base.LazyValue{
+		ValueOrHandle: append(make([]byte, 0, len(value.ValueOrHandle)), value.ValueOrHandle...),
+	}
+	if value.Fetcher != nil {
+		fetcher := new(base.LazyFetcher)
+		*fetcher = *value.Fetcher
+		i.lastValue.Fetcher = fetcher
+	}
+	return i.lastKey, i.lastValue
+}
+
+func (i *iter) trashLastKV() {
+	if i.lastKey == nil {
+		return
+	}
+	if i.ignoreKinds[i.lastKey.Kind()] {
+		return
+	}
+
+	if i.lastKey != nil {
+		for j := range i.lastKey.UserKey {
+			i.lastKey.UserKey[j] = 0xff
+		}
+		i.lastKey.Trailer = 0xffffffffffffffff
+	}
+	for j := range i.lastValue.ValueOrHandle {
+		i.lastValue.ValueOrHandle[j] = 0xff
+	}
+	if i.lastValue.Fetcher != nil {
+		// Not all the LazyFetcher fields are visible, so we zero out the last
+		// value's Fetcher struct entirely.
+		*i.lastValue.Fetcher = base.LazyFetcher{}
+	}
+}
+
+func (i *iter) SeekGE(key []byte, flags base.SeekGEFlags) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekGE(key, flags))
+}
+
+func (i *iter) SeekPrefixGE(
+	prefix, key []byte, flags base.SeekGEFlags,
+) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekPrefixGE(prefix, key, flags))
+}
+
+func (i *iter) SeekLT(key []byte, flags base.SeekLTFlags) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.SeekLT(key, flags))
+}
+
+func (i *iter) First() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.First())
+}
+
+func (i *iter) Last() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Last())
+}
+
+func (i *iter) Next() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Next())
+}
+
+func (i *iter) Prev() (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.Prev())
+}
+
+func (i *iter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) {
+	return i.update(i.iter.NextPrefix(succKey))
+}
+
+func (i *iter) Error() error {
+	if err := i.iter.Error(); err != nil {
+		return err
+	}
+	return i.err
+}
+
+func (i *iter) Close() error {
+	return i.iter.Close()
+}
+
+func (i *iter) SetBounds(lower, upper []byte) {
+	i.iter.SetBounds(lower, upper)
+}
+
+func (i *iter) String() string {
+	return i.iter.String()
+}

vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go

@@ -228,6 +228,14 @@ func (i *DefragmentingIter) SeekGE(key []byte) *Span {
 	// fragments before this span. Defragment backward to find the start of the
 	// defragmented span.
 	i.defragmentBackward()
+
+	// Defragmenting backward may have stopped because it encountered an error.
+	// If so, we must not continue so that i.iter.Error() (and thus i.Error())
+	// yields the error.
+	if i.iterSpan == nil && i.iter.Error() != nil {
+		return nil
+	}
+
 	if i.iterPos == iterPosPrev {
 		// Next once back onto the span.
 		i.iterSpan = i.iter.Next()
@@ -259,6 +267,14 @@ func (i *DefragmentingIter) SeekLT(key []byte) *Span {
 	// fragments after this span. Defragment forward to find the end of the
 	// defragmented span.
 	i.defragmentForward()
+
+	// Defragmenting forward may have stopped because it encountered an error.
+	// If so, we must not continue so that i.iter.Error() (and thus i.Error())
+	// yields the error.
+	if i.iterSpan == nil && i.iter.Error() != nil {
+		return nil
+	}
+
 	if i.iterPos == iterPosNext {
 		// Prev once back onto the span.
 		i.iterSpan = i.iter.Prev()
@@ -304,7 +320,7 @@ func (i *DefragmentingIter) Next() *Span {
 		// Next once to move onto y, defragment forward to land on the first z
 		// position.
 		i.iterSpan = i.iter.Next()
-		if invariants.Enabled && i.iterSpan == nil {
+		if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
 			panic("pebble: invariant violation: no next span while switching directions")
 		}
 		// We're now positioned on the first span that was defragmented into the
@@ -380,7 +396,7 @@ func (i *DefragmentingIter) Prev() *Span {
 		// Prev once to move onto y, defragment backward to land on the last x
 		// position.
 		i.iterSpan = i.iter.Prev()
-		if invariants.Enabled && i.iterSpan == nil {
+		if invariants.Enabled && i.iterSpan == nil && i.iter.Error() == nil {
 			panic("pebble: invariant violation: no previous span while switching directions")
 		}
 		// We're now positioned on the last span that was defragmented into the
@@ -437,6 +453,15 @@ func (i *DefragmentingIter) defragmentForward() *Span {
 		i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
 		i.iterSpan = i.iter.Next()
 	}
+	// i.iterSpan == nil
+	//
+	// The inner iterator may return nil when it encounters an error. If there
+	// was an error, we don't know whether there is another span we should
+	// defragment or not. Return nil so that the caller knows they should check
+	// Error().
+	if i.iter.Error() != nil {
+		return nil
+	}
 	i.curr.Keys = i.keysBuf
 	return &i.curr
 }
@@ -470,6 +495,15 @@ func (i *DefragmentingIter) defragmentBackward() *Span {
 		i.keysBuf = i.reduce(i.keysBuf, i.iterSpan.Keys)
 		i.iterSpan = i.iter.Prev()
 	}
+	// i.iterSpan == nil
+	//
+	// The inner iterator may return nil when it encounters an error. If there
+	// was an error, we don't know whether there is another span we should
+	// defragment or not. Return nil so that the caller knows they should check
+	// Error().
+	if i.iter.Error() != nil {
+		return nil
+	}
 	i.curr.Keys = i.keysBuf
 	return &i.curr
 }

vendor/github.com/cockroachdb/pebble/internal/keyspan/filter.go

@@ -4,6 +4,8 @@
 
 package keyspan
 
+import "github.com/cockroachdb/pebble/internal/base"
+
 // FilterFunc defines a transform from the input Span into the output Span. The
 // function returns true if the Span should be returned by the iterator, and
 // false if the Span should be skipped. The FilterFunc is permitted to mutate
@@ -21,6 +23,7 @@ type FilterFunc func(in *Span, out *Span) (keep bool)
 type filteringIter struct {
 	iter     FragmentIterator
 	filterFn FilterFunc
+	cmp      base.Compare
 
 	// span is a mutable Span passed to the filterFn. The filterFn is free to
 	// mutate this Span. The slice of Keys in the Span is reused with every call
@@ -32,18 +35,32 @@ var _ FragmentIterator = (*filteringIter)(nil)
 
 // Filter returns a new filteringIter that will filter the Spans from the
 // provided child iterator using the provided FilterFunc.
-func Filter(iter FragmentIterator, filter FilterFunc) FragmentIterator {
+func Filter(iter FragmentIterator, filter FilterFunc, cmp base.Compare) FragmentIterator {
-	return &filteringIter{iter: iter, filterFn: filter}
+	return &filteringIter{iter: iter, filterFn: filter, cmp: cmp}
 }
 
 // SeekGE implements FragmentIterator.
 func (i *filteringIter) SeekGE(key []byte) *Span {
-	return i.filter(i.iter.SeekGE(key), +1)
+	span := i.filter(i.iter.SeekGE(key), +1)
+	// i.filter could return a span that's less than key, _if_ the filterFunc
+	// (which has no knowledge of the seek key) mutated the span to end at a key
+	// less than or equal to `key`. Detect this case and next/invalidate the iter.
+	if span != nil && i.cmp(span.End, key) <= 0 {
+		return i.Next()
+	}
+	return span
 }
 
 // SeekLT implements FragmentIterator.
 func (i *filteringIter) SeekLT(key []byte) *Span {
-	return i.filter(i.iter.SeekLT(key), -1)
+	span := i.filter(i.iter.SeekLT(key), -1)
+	// i.filter could return a span that's >= key, _if_ the filterFunc (which has
+	// no knowledge of the seek key) mutated the span to start at a key greater
+	// than or equal to `key`. Detect this case and prev/invalidate the iter.
+	if span != nil && i.cmp(span.Start, key) >= 0 {
+		return i.Prev()
+	}
+	return span
 }
 
 // First implements FragmentIterator.

vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go

@@ -7,6 +7,7 @@ package keyspan
 import (
 	"fmt"
 
+	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
 )
@@ -111,6 +112,22 @@ type InterleavingIter struct {
 	nextPrefixBuf []byte
 	pointKey      *base.InternalKey
 	pointVal      base.LazyValue
+	// err holds an iterator error from either pointIter or keyspanIter. It's
+	// reset to nil on seeks. An overview of error-handling mechanics:
+	//
+	// Whenever either pointIter or keyspanIter is respositioned and a nil
+	// key/span is returned, the code performing the positioning is responsible
+	// for checking the iterator's Error() value. This happens in savePoint and
+	// saveSpan[Forward,Backward].
+	//
+	// Once i.err is non-nil, the computation of i.pos must set i.pos =
+	// posExhausted. This happens in compute[Smallest|Largest]Pos and
+	// [next|prev]Pos. Setting i.pos to posExhausted ensures we'll yield nil to
+	// the caller, which they'll interpret as a signal they must check Error().
+	//
+	// INVARIANTS:
+	// i.err != nil => i.pos = posExhausted
+	err error
 	// prefix records the iterator's current prefix if the iterator is in prefix
 	// mode. During prefix mode, Pebble will truncate spans to the next prefix.
 	// If the iterator subsequently leaves prefix mode, the existing span cached
@@ -216,7 +233,7 @@ func (i *InterleavingIter) InitSeekGE(
 	i.dir = +1
 	i.clearMask()
 	i.prefix = prefix
-	i.pointKey, i.pointVal = pointKey, pointValue
+	i.savePoint(pointKey, pointValue)
 	// NB: This keyspanSeekGE call will truncate the span to the seek key if
 	// necessary. This truncation is important for cases where a switch to
 	// combined iteration is made during a user-initiated SeekGE.
@@ -240,7 +257,7 @@ func (i *InterleavingIter) InitSeekLT(
 ) (*base.InternalKey, base.LazyValue) {
 	i.dir = -1
 	i.clearMask()
-	i.pointKey, i.pointVal = pointKey, pointValue
+	i.savePoint(pointKey, pointValue)
 	i.keyspanSeekLT(key)
 	i.computeLargestPos()
 	return i.yieldPosition(i.lower, i.prevPos)
@@ -260,9 +277,10 @@ func (i *InterleavingIter) InitSeekLT(
 func (i *InterleavingIter) SeekGE(
 	key []byte, flags base.SeekGEFlags,
 ) (*base.InternalKey, base.LazyValue) {
+	i.err = nil
 	i.clearMask()
 	i.disablePrefixMode()
-	i.pointKey, i.pointVal = i.pointIter.SeekGE(key, flags)
+	i.savePoint(i.pointIter.SeekGE(key, flags))
 
 	// We need to seek the keyspan iterator too. If the keyspan iterator was
 	// already positioned at a span, we might be able to avoid the seek if the
@@ -270,7 +288,7 @@ func (i *InterleavingIter) SeekGE(
 	if i.span != nil && i.cmp(key, i.span.End) < 0 && i.cmp(key, i.span.Start) >= 0 {
 		// We're seeking within the existing span's bounds. We still might need
 		// truncate the span to the iterator's bounds.
-		i.checkForwardBound()
+		i.saveSpanForward(i.span)
 		i.savedKeyspan()
 	} else {
 		i.keyspanSeekGE(key, nil /* prefix */)
@@ -295,9 +313,10 @@ func (i *InterleavingIter) SeekGE(
 func (i *InterleavingIter) SeekPrefixGE(
 	prefix, key []byte, flags base.SeekGEFlags,
 ) (*base.InternalKey, base.LazyValue) {
+	i.err = nil
 	i.clearMask()
 	i.prefix = prefix
-	i.pointKey, i.pointVal = i.pointIter.SeekPrefixGE(prefix, key, flags)
+	i.savePoint(i.pointIter.SeekPrefixGE(prefix, key, flags))
 
 	// We need to seek the keyspan iterator too. If the keyspan iterator was
 	// already positioned at a span, we might be able to avoid the seek if the
@@ -326,7 +345,7 @@ func (i *InterleavingIter) SeekPrefixGE(
 		if ei := i.comparer.Split(i.span.End); i.cmp(prefix, i.span.End[:ei]) < 0 {
 			// We're seeking within the existing span's bounds. We still might need
 			// truncate the span to the iterator's bounds.
-			i.checkForwardBound()
+			i.saveSpanForward(i.span)
 			i.savedKeyspan()
 			seekKeyspanIter = false
 		}
@@ -344,9 +363,10 @@ func (i *InterleavingIter) SeekPrefixGE(
 func (i *InterleavingIter) SeekLT(
 	key []byte, flags base.SeekLTFlags,
 ) (*base.InternalKey, base.LazyValue) {
+	i.err = nil
 	i.clearMask()
 	i.disablePrefixMode()
-	i.pointKey, i.pointVal = i.pointIter.SeekLT(key, flags)
+	i.savePoint(i.pointIter.SeekLT(key, flags))
 
 	// We need to seek the keyspan iterator too. If the keyspan iterator was
 	// already positioned at a span, we might be able to avoid the seek if the
@@ -354,13 +374,13 @@ func (i *InterleavingIter) SeekLT(
 	if i.span != nil && i.cmp(key, i.span.Start) > 0 && i.cmp(key, i.span.End) < 0 {
 		// We're seeking within the existing span's bounds. We still might need
 		// truncate the span to the iterator's bounds.
-		i.checkBackwardBound()
+		i.saveSpanBackward(i.span)
 		// The span's start key is still not guaranteed to be less than key,
 		// because of the bounds enforcement. Consider the following example:
 		//
 		// Bounds are set to [d,e). The user performs a SeekLT(d). The
 		// FragmentIterator.SeekLT lands on a span [b,f). This span has a start
-		// key less than d, as expected. Above, checkBackwardBound truncates the
+		// key less than d, as expected. Above, saveSpanBackward truncates the
 		// span to match the iterator's current bounds, modifying the span to
 		// [d,e), which does not overlap the search space of [-∞, d).
 		//
@@ -382,11 +402,11 @@ func (i *InterleavingIter) SeekLT(
 
 // First implements (base.InternalIterator).First.
 func (i *InterleavingIter) First() (*base.InternalKey, base.LazyValue) {
+	i.err = nil
 	i.clearMask()
 	i.disablePrefixMode()
-	i.pointKey, i.pointVal = i.pointIter.First()
+	i.savePoint(i.pointIter.First())
-	i.span = i.keyspanIter.First()
+	i.saveSpanForward(i.keyspanIter.First())
-	i.checkForwardBound()
 	i.savedKeyspan()
 	i.dir = +1
 	i.computeSmallestPos()
@@ -395,11 +415,11 @@ func (i *InterleavingIter) First() (*base.InternalKey, base.LazyValue) {
 
 // Last implements (base.InternalIterator).Last.
 func (i *InterleavingIter) Last() (*base.InternalKey, base.LazyValue) {
+	i.err = nil
 	i.clearMask()
 	i.disablePrefixMode()
-	i.pointKey, i.pointVal = i.pointIter.Last()
+	i.savePoint(i.pointIter.Last())
-	i.span = i.keyspanIter.Last()
+	i.saveSpanBackward(i.keyspanIter.Last())
-	i.checkBackwardBound()
 	i.savedKeyspan()
 	i.dir = -1
 	i.computeLargestPos()
@@ -436,8 +456,7 @@ func (i *InterleavingIter) Next() (*base.InternalKey, base.LazyValue) {
 			// entirely behind the current key (!i.withinSpan), then we
 			// need to move it to the first span in the forward direction.
 			if !i.withinSpan {
-				i.span = i.keyspanIter.Next()
+				i.saveSpanForward(i.keyspanIter.Next())
-				i.checkForwardBound()
 				i.savedKeyspan()
 			}
 		case posKeyspanStart:
@@ -445,12 +464,12 @@ func (i *InterleavingIter) Next() (*base.InternalKey, base.LazyValue) {
 			// Since we're positioned on a Span, the pointIter is positioned
 			// entirely behind the current iterator position. Reposition it
 			// ahead of the current iterator position.
-			i.pointKey, i.pointVal = i.pointIter.Next()
+			i.savePoint(i.pointIter.Next())
 		case posKeyspanEnd:
 			// Since we're positioned on a Span, the pointIter is positioned
 			// entirely behind of the current iterator position. Reposition it
 			// ahead the current iterator position.
-			i.pointKey, i.pointVal = i.pointIter.Next()
+			i.savePoint(i.pointIter.Next())
 		}
 		// Fallthrough to calling i.nextPos.
 	}
@@ -468,7 +487,7 @@ func (i *InterleavingIter) NextPrefix(succKey []byte) (*base.InternalKey, base.L
 	case posExhausted:
 		return nil, base.LazyValue{}
 	case posPointKey:
-		i.pointKey, i.pointVal = i.pointIter.NextPrefix(succKey)
+		i.savePoint(i.pointIter.NextPrefix(succKey))
 		if i.withinSpan {
 			if i.pointKey == nil || i.cmp(i.span.End, i.pointKey.UserKey) <= 0 {
 				i.pos = posKeyspanEnd
@@ -514,15 +533,14 @@ func (i *InterleavingIter) Prev() (*base.InternalKey, base.LazyValue) {
 			// current key (!i.withinSpan), then we need to move it to the first
 			// span in the reverse direction.
 			if !i.withinSpan {
-				i.span = i.keyspanIter.Prev()
+				i.saveSpanBackward(i.keyspanIter.Prev())
-				i.checkBackwardBound()
 				i.savedKeyspan()
 			}
 		case posKeyspanStart:
 			// Since we're positioned on a Span, the pointIter is positioned
 			// entirely ahead of the current iterator position. Reposition it
 			// behind the current iterator position.
-			i.pointKey, i.pointVal = i.pointIter.Prev()
+			i.savePoint(i.pointIter.Prev())
 			// Without considering truncation of spans to seek keys, the keyspan
 			// iterator is already in the right place. But consider span [a, z)
 			// and this sequence of iterator calls:
@@ -544,7 +562,7 @@ func (i *InterleavingIter) Prev() (*base.InternalKey, base.LazyValue) {
 			// Since we're positioned on a Span, the pointIter is positioned
 			// entirely ahead of the current iterator position. Reposition it
 			// behind the current iterator position.
-			i.pointKey, i.pointVal = i.pointIter.Prev()
+			i.savePoint(i.pointIter.Prev())
 		}
 
 		if i.spanMarkerTruncated {
@@ -561,15 +579,17 @@ func (i *InterleavingIter) Prev() (*base.InternalKey, base.LazyValue) {
 //
 //	MIN(i.pointKey, i.span.Start)
 func (i *InterleavingIter) computeSmallestPos() {
-	if i.span != nil && (i.pointKey == nil || i.cmp(i.startKey(), i.pointKey.UserKey) <= 0) {
+	if i.err == nil {
-		i.withinSpan = true
+		if i.span != nil && (i.pointKey == nil || i.cmp(i.startKey(), i.pointKey.UserKey) <= 0) {
-		i.pos = posKeyspanStart
+			i.withinSpan = true
-		return
+			i.pos = posKeyspanStart
-	}
+			return
-	i.withinSpan = false
+		}
-	if i.pointKey != nil {
+		i.withinSpan = false
-		i.pos = posPointKey
+		if i.pointKey != nil {
-		return
+			i.pos = posPointKey
+			return
+		}
 	}
 	i.pos = posExhausted
 }
@@ -578,30 +598,52 @@ func (i *InterleavingIter) computeSmallestPos() {
 //
 //	MAX(i.pointKey, i.span.End)
 func (i *InterleavingIter) computeLargestPos() {
-	if i.span != nil && (i.pointKey == nil || i.cmp(i.span.End, i.pointKey.UserKey) > 0) {
+	if i.err == nil {
-		i.withinSpan = true
+		if i.span != nil && (i.pointKey == nil || i.cmp(i.span.End, i.pointKey.UserKey) > 0) {
-		i.pos = posKeyspanEnd
+			i.withinSpan = true
-		return
+			i.pos = posKeyspanEnd
-	}
+			return
-	i.withinSpan = false
+		}
-	if i.pointKey != nil {
+		i.withinSpan = false
-		i.pos = posPointKey
+		if i.pointKey != nil {
-		return
+			i.pos = posPointKey
+			return
+		}
 	}
 	i.pos = posExhausted
 }
 
 // nextPos advances the iterator one position in the forward direction.
 func (i *InterleavingIter) nextPos() {
+	if invariants.Enabled {
+		defer func() {
+			if i.err != nil && i.pos != posExhausted {
+				panic(errors.AssertionFailedf("iterator has accumulated error but i.pos = %d", i.pos))
+			}
+		}()
+	}
+	// NB: If i.err != nil or any of the positioning methods performed in this
+	// function result in i.err != nil, we must set i.pos = posExhausted. We
+	// perform this check explicitly here, but if any of the branches below
+	// advance either iterator, they must also check i.err and set posExhausted
+	// if necessary.
+	if i.err != nil {
+		i.pos = posExhausted
+		return
+	}
+
 	switch i.pos {
 	case posExhausted:
-		i.pointKey, i.pointVal = i.pointIter.Next()
+		i.savePoint(i.pointIter.Next())
-		i.span = i.keyspanIter.Next()
+		i.saveSpanForward(i.keyspanIter.Next())
-		i.checkForwardBound()
 		i.savedKeyspan()
 		i.computeSmallestPos()
 	case posPointKey:
-		i.pointKey, i.pointVal = i.pointIter.Next()
+		i.savePoint(i.pointIter.Next())
+		if i.err != nil {
+			i.pos = posExhausted
+			return
+		}
 		// If we're not currently within the span, we want to chose the
 		// MIN(pointKey,span.Start), which is exactly the calculation performed
 		// by computeSmallestPos.
@@ -635,26 +677,45 @@ func (i *InterleavingIter) nextPos() {
 			i.pos = posKeyspanEnd
 		}
 	case posKeyspanEnd:
-		i.span = i.keyspanIter.Next()
+		i.saveSpanForward(i.keyspanIter.Next())
-		i.checkForwardBound()
 		i.savedKeyspan()
 		i.computeSmallestPos()
 	default:
-		panic(fmt.Sprintf("unexpected pos=%d\n", i.pos))
+		panic(fmt.Sprintf("unexpected pos=%d", i.pos))
 	}
 }
 
 // prevPos advances the iterator one position in the reverse direction.
 func (i *InterleavingIter) prevPos() {
+	if invariants.Enabled {
+		defer func() {
+			if i.err != nil && i.pos != posExhausted {
+				panic(errors.AssertionFailedf("iterator has accumulated error but i.pos = %d", i.pos))
+			}
+		}()
+	}
+	// NB: If i.err != nil or any of the positioning methods performed in this
+	// function result in i.err != nil, we must set i.pos = posExhausted. We
+	// perform this check explicitly here, but if any of the branches below
+	// advance either iterator, they must also check i.err and set posExhausted
+	// if necessary.
+	if i.err != nil {
+		i.pos = posExhausted
+		return
+	}
+
 	switch i.pos {
 	case posExhausted:
-		i.pointKey, i.pointVal = i.pointIter.Prev()
+		i.savePoint(i.pointIter.Prev())
-		i.span = i.keyspanIter.Prev()
+		i.saveSpanBackward(i.keyspanIter.Prev())
-		i.checkBackwardBound()
 		i.savedKeyspan()
 		i.computeLargestPos()
 	case posPointKey:
-		i.pointKey, i.pointVal = i.pointIter.Prev()
+		i.savePoint(i.pointIter.Prev())
+		if i.err != nil {
+			i.pos = posExhausted
+			return
+		}
 		// If we're not currently covered by the span, we want to chose the
 		// MAX(pointKey,span.End), which is exactly the calculation performed
 		// by computeLargestPos.
@@ -676,8 +737,7 @@ func (i *InterleavingIter) prevPos() {
 			}
 		}
 	case posKeyspanStart:
-		i.span = i.keyspanIter.Prev()
+		i.saveSpanBackward(i.keyspanIter.Prev())
-		i.checkBackwardBound()
 		i.savedKeyspan()
 		i.computeLargestPos()
 	case posKeyspanEnd:
@@ -688,7 +748,7 @@ func (i *InterleavingIter) prevPos() {
 			i.pos = posKeyspanStart
 		}
 	default:
-		panic(fmt.Sprintf("unexpected pos=%d\n", i.pos))
+		panic(fmt.Sprintf("unexpected pos=%d", i.pos))
 	}
 }
 
@@ -760,21 +820,19 @@ func (i *InterleavingIter) yieldPosition(
 
 // keyspanSeekGE seeks the keyspan iterator to the first span covering a key ≥ k.
 func (i *InterleavingIter) keyspanSeekGE(k []byte, prefix []byte) {
-	i.span = i.keyspanIter.SeekGE(k)
+	i.saveSpanForward(i.keyspanIter.SeekGE(k))
-	i.checkForwardBound()
 	i.savedKeyspan()
 }
 
 // keyspanSeekLT seeks the keyspan iterator to the last span covering a key < k.
 func (i *InterleavingIter) keyspanSeekLT(k []byte) {
-	i.span = i.keyspanIter.SeekLT(k)
+	i.saveSpanBackward(i.keyspanIter.SeekLT(k))
-	i.checkBackwardBound()
 	// The current span's start key is not guaranteed to be less than key,
 	// because of the bounds enforcement. Consider the following example:
 	//
 	// Bounds are set to [d,e). The user performs a SeekLT(d). The
 	// FragmentIterator.SeekLT lands on a span [b,f). This span has a start key
-	// less than d, as expected. Above, checkBackwardBound truncates the span to
+	// less than d, as expected. Above, saveSpanBackward truncates the span to
 	// match the iterator's current bounds, modifying the span to [d,e), which
 	// does not overlap the search space of [-∞, d).
 	//
@@ -786,12 +844,21 @@ func (i *InterleavingIter) keyspanSeekLT(k []byte) {
 	i.savedKeyspan()
 }
 
-func (i *InterleavingIter) checkForwardBound() {
+func (i *InterleavingIter) saveSpanForward(span *Span) {
+	i.span = span
 	i.truncated = false
 	i.truncatedSpan = Span{}
 	if i.span == nil {
+		i.err = firstError(i.err, i.keyspanIter.Error())
 		return
 	}
+	if invariants.Enabled {
+		if err := i.keyspanIter.Error(); err != nil {
+			panic(errors.WithSecondaryError(
+				errors.AssertionFailedf("pebble: %T keyspan iterator returned non-nil span %s while iter has error", i.keyspanIter, i.span),
+				err))
+		}
+	}
 	// Check the upper bound if we have one.
 	if i.upper != nil && i.cmp(i.span.Start, i.upper) >= 0 {
 		i.span = nil
@@ -839,12 +906,22 @@ func (i *InterleavingIter) checkForwardBound() {
 	}
 }
 
-func (i *InterleavingIter) checkBackwardBound() {
+func (i *InterleavingIter) saveSpanBackward(span *Span) {
+	i.span = span
 	i.truncated = false
 	i.truncatedSpan = Span{}
 	if i.span == nil {
+		i.err = firstError(i.err, i.keyspanIter.Error())
 		return
 	}
+	if invariants.Enabled {
+		if err := i.keyspanIter.Error(); err != nil {
+			panic(errors.WithSecondaryError(
+				errors.AssertionFailedf("pebble: %T keyspan iterator returned non-nil span %s while iter has error", i.keyspanIter, i.span),
+				err))
+		}
+	}
+
 	// Check the lower bound if we have one.
 	if i.lower != nil && i.cmp(i.span.End, i.lower) <= 0 {
 		i.span = nil
@@ -947,6 +1024,12 @@ func (i *InterleavingIter) verify(
 			panic("pebble: invariant violation: key < lower bound")
 		case k != nil && i.upper != nil && i.cmp(k.UserKey, i.upper) >= 0:
 			panic("pebble: invariant violation: key ≥ upper bound")
+		case i.err != nil && k != nil:
+			panic("pebble: invariant violation: accumulated error swallowed")
+		case i.err == nil && i.pointIter.Error() != nil:
+			panic("pebble: invariant violation: pointIter swallowed")
+		case i.err == nil && i.keyspanIter.Error() != nil:
+			panic("pebble: invariant violation: keyspanIter error swallowed")
 		}
 	}
 	return k, v
@@ -990,6 +1073,20 @@ func (i *InterleavingIter) startKey() []byte {
 	return i.span.Start
 }
 
+func (i *InterleavingIter) savePoint(key *base.InternalKey, value base.LazyValue) {
+	i.pointKey, i.pointVal = key, value
+	if key == nil {
+		i.err = firstError(i.err, i.pointIter.Error())
+	}
+	if invariants.Enabled {
+		if err := i.pointIter.Error(); key != nil && err != nil {
+			panic(errors.WithSecondaryError(
+				errors.AssertionFailedf("pebble: %T point iterator returned non-nil key %q while iter has error", i.pointIter, key),
+				err))
+		}
+	}
+}
+
 // Span returns the span covering the last key returned, if any. A span key is
 // considered to 'cover' a key if the key falls within the span's user key
 // bounds. The returned span is owned by the InterleavingIter. The caller is
@@ -1023,7 +1120,7 @@ func (i *InterleavingIter) Invalidate() {
 
 // Error implements (base.InternalIterator).Error.
 func (i *InterleavingIter) Error() error {
-	return firstError(i.pointIter.Error(), i.keyspanIter.Error())
+	return i.err
 }
 
 // Close implements (base.InternalIterator).Close.

vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go

@@ -51,6 +51,8 @@ type FragmentIterator interface {
 	Prev() *Span
 
 	// Error returns any accumulated error.
+	//
+	// TODO(jackson): Lift errors into return values on the positioning methods.
 	Error() error
 
 	// Close closes the iterator and returns any accumulated error. Exhausting

vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go

@@ -69,5 +69,5 @@ func Truncate(
 		}
 
 		return !out.Empty() && cmp(out.Start, out.End) < 0
-	})
+	}, cmp)
 }

vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go

@@ -195,7 +195,7 @@ func (n *node) incRef() {
 // new nodes pass contentsToo=false to preserve existing reference counts during
 // operations that should yield a net-zero change to descendant refcounts.
 // When a node is released, its contained files are dereferenced.
-func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
+func (n *node) decRef(contentsToo bool, obsolete *[]*FileBacking) {
 	if n.ref.Add(-1) > 0 {
 		// Other references remain. Can't free.
 		return
@@ -215,7 +215,13 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) {
 				if obsolete == nil {
 					panic(fmt.Sprintf("file metadata %s dereferenced to zero during tree mutation", f.FileNum))
 				}
-				*obsolete = append(*obsolete, f)
+				// Reference counting is performed on the FileBacking. In the case
+				// of a virtual sstable, this reference counting is performed on
+				// a FileBacking which is shared by every single virtual sstable
+				// with the same backing sstable. If the reference count hits 0,
+				// then we know that the FileBacking won't be required by any
+				// sstable in Pebble, and that the backing sstable can be deleted.
+				*obsolete = append(*obsolete, f.FileBacking)
 			}
 		}
 		if !n.leaf {
@@ -761,8 +767,8 @@ type btree struct {
 
 // Release dereferences and clears the root node of the btree, removing all
 // items from the btree. In doing so, it decrements contained file counts.
-// It returns a slice of newly obsolete files, if any.
+// It returns a slice of newly obsolete backing files, if any.
-func (t *btree) Release() (obsolete []*FileMetadata) {
+func (t *btree) Release() (obsolete []*FileBacking) {
 	if t.root != nil {
 		t.root.decRef(true /* contentsToo */, &obsolete)
 		t.root = nil

vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go

@@ -17,20 +17,26 @@ import (
 type LevelMetadata struct {
 	level     int
 	totalSize uint64
-	tree      btree
+	// NumVirtual is the number of virtual sstables in the level.
+	NumVirtual uint64
+	// VirtualSize is the size of the virtual sstables in the level.
+	VirtualSize uint64
+	tree        btree
 }
 
 // clone makes a copy of the level metadata, implicitly increasing the ref
 // count of every file contained within lm.
 func (lm *LevelMetadata) clone() LevelMetadata {
 	return LevelMetadata{
-		level:     lm.level,
+		level:       lm.level,
-		totalSize: lm.totalSize,
+		totalSize:   lm.totalSize,
-		tree:      lm.tree.Clone(),
+		NumVirtual:  lm.NumVirtual,
+		VirtualSize: lm.VirtualSize,
+		tree:        lm.tree.Clone(),
 	}
 }
 
-func (lm *LevelMetadata) release() (obsolete []*FileMetadata) {
+func (lm *LevelMetadata) release() (obsolete []*FileBacking) {
 	return lm.tree.Release()
 }
 
@@ -44,6 +50,10 @@ func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetad
 	lm.tree, _ = makeBTree(bcmp, files)
 	for _, f := range files {
 		lm.totalSize += f.Size
+		if f.Virtual {
+			lm.NumVirtual++
+			lm.VirtualSize += f.Size
+		}
 	}
 	return lm
 }
@@ -62,11 +72,19 @@ func (lm *LevelMetadata) insert(f *FileMetadata) error {
 		return err
 	}
 	lm.totalSize += f.Size
+	if f.Virtual {
+		lm.NumVirtual++
+		lm.VirtualSize += f.Size
+	}
 	return nil
 }
 
 func (lm *LevelMetadata) remove(f *FileMetadata) bool {
 	lm.totalSize -= f.Size
+	if f.Virtual {
+		lm.NumVirtual--
+		lm.VirtualSize -= f.Size
+	}
 	return lm.tree.Delete(f)
 }
 
@@ -303,6 +321,32 @@ func (ls *LevelSlice) SizeSum() uint64 {
 	return sum
 }
 
+// NumVirtual returns the number of virtual sstables in the level. Its runtime is
+// linear in the length of the slice.
+func (ls *LevelSlice) NumVirtual() uint64 {
+	var n uint64
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		if f.Virtual {
+			n++
+		}
+	}
+	return n
+}
+
+// VirtualSizeSum returns the sum of the sizes of the virtual sstables in the
+// level.
+func (ls *LevelSlice) VirtualSizeSum() uint64 {
+	var sum uint64
+	iter := ls.Iter()
+	for f := iter.First(); f != nil; f = iter.Next() {
+		if f.Virtual {
+			sum += f.Size
+		}
+	}
+	return sum
+}
+
 // Reslice constructs a new slice backed by the same underlying level, with
 // new start and end positions. Reslice invokes the provided function, passing
 // two LevelIterators: one positioned to i's inclusive start and one

vendor/github.com/cockroachdb/pebble/internal/manifest/version.go

@@ -49,6 +49,9 @@ type TableStats struct {
 	// The number of point and range deletion entries in the table.
 	NumDeletions uint64
 	// NumRangeKeySets is the total number of range key sets in the table.
+	//
+	// NB: If there's a chance that the sstable contains any range key sets,
+	// then NumRangeKeySets must be > 0.
 	NumRangeKeySets uint64
 	// Estimate of the total disk space that may be dropped by this table's
 	// point deletions by compacting them.
@@ -169,22 +172,13 @@ type FileMetadata struct {
 	// FileNum is the file number.
 	//
 	// INVARIANT: when !FileMetadata.Virtual, FileNum == FileBacking.DiskFileNum.
-	//
-	// TODO(bananabrick): Consider creating separate types for
-	// FileMetadata.FileNum and FileBacking.FileNum. FileNum is used both as
-	// an indentifier for the FileMetadata in Pebble, and also as a handle to
-	// perform reads and writes. We should ensure through types that
-	// FileMetadata.FileNum isn't used to perform reads, and that
-	// FileBacking.FileNum isn't used as an identifier for the FileMetadata.
 	FileNum base.FileNum
 	// Size is the size of the file, in bytes. Size is an approximate value for
 	// virtual sstables.
 	//
-	// INVARIANT: when !FileMetadata.Virtual, Size == FileBacking.Size.
+	// INVARIANTS:
-	//
+	// - When !FileMetadata.Virtual, Size == FileBacking.Size.
-	// TODO(bananabrick): Size is currently used in metrics, and for many key
+	// - Size should be non-zero. Size 0 virtual sstables must not be created.
-	// Pebble level heuristics. Make sure that the heuristics will still work
-	// appropriately with an approximate value of size.
 	Size uint64
 	// File creation time in seconds since the epoch (1970-01-01 00:00:00
 	// UTC). For ingested sstables, this corresponds to the time the file was
@@ -309,6 +303,7 @@ type PhysicalFileMeta struct {
 //     The underlying file's size is stored in FileBacking.Size, though it could
 //     also be estimated or could correspond to just the referenced portion of
 //     a file (eg. if the file originated on another node).
+//   - Size must be > 0.
 //   - SmallestSeqNum and LargestSeqNum are loose bounds for virtual sstables.
 //     This means that all keys in the virtual sstable must have seqnums within
 //     [SmallestSeqNum, LargestSeqNum], however there's no guarantee that there's
@@ -1209,12 +1204,7 @@ func (v *Version) Unref() {
 		l := v.list
 		l.mu.Lock()
 		l.Remove(v)
-		obsolete := v.unrefFiles()
+		v.Deleted(v.unrefFiles())
-		fileBacking := make([]*FileBacking, len(obsolete))
-		for i, f := range obsolete {
-			fileBacking[i] = f.FileBacking
-		}
-		v.Deleted(fileBacking)
 		l.mu.Unlock()
 	}
 }
@@ -1226,17 +1216,12 @@ func (v *Version) Unref() {
 func (v *Version) UnrefLocked() {
 	if v.refs.Add(-1) == 0 {
 		v.list.Remove(v)
-		obsolete := v.unrefFiles()
+		v.Deleted(v.unrefFiles())
-		fileBacking := make([]*FileBacking, len(obsolete))
-		for i, f := range obsolete {
-			fileBacking[i] = f.FileBacking
-		}
-		v.Deleted(fileBacking)
 	}
 }
 
-func (v *Version) unrefFiles() []*FileMetadata {
+func (v *Version) unrefFiles() []*FileBacking {
-	var obsolete []*FileMetadata
+	var obsolete []*FileBacking
 	for _, lm := range v.Levels {
 		obsolete = append(obsolete, lm.release()...)
 	}
@@ -1368,16 +1353,20 @@ func (v *Version) Overlaps(
 // CheckOrdering checks that the files are consistent with respect to
 // increasing file numbers (for level 0 files) and increasing and non-
 // overlapping internal key ranges (for level non-0 files).
-func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error {
+func (v *Version) CheckOrdering(
+	cmp Compare, format base.FormatKey, order OrderingInvariants,
+) error {
 	for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- {
 		sublevelIter := v.L0SublevelFiles[sublevel].Iter()
-		if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter); err != nil {
+		// Sublevels have NEVER allowed split user keys, so we can pass
+		// ProhibitSplitUserKeys.
+		if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter, ProhibitSplitUserKeys); err != nil {
 			return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
 		}
 	}
 
 	for level, lm := range v.Levels {
-		if err := CheckOrdering(cmp, format, Level(level), lm.Iter()); err != nil {
+		if err := CheckOrdering(cmp, format, Level(level), lm.Iter(), order); err != nil {
 			return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format))
 		}
 	}
@@ -1446,10 +1435,34 @@ func (l *VersionList) Remove(v *Version) {
 	v.list = nil // avoid memory leaks
 }
 
+// OrderingInvariants dictates the file ordering invariants active.
+type OrderingInvariants int8
+
+const (
+	// ProhibitSplitUserKeys indicates that adjacent files within a level cannot
+	// contain the same user key.
+	ProhibitSplitUserKeys OrderingInvariants = iota
+	// AllowSplitUserKeys indicates that adjacent files within a level may
+	// contain the same user key. This is only allowed by historical format
+	// major versions.
+	//
+	// TODO(jackson): Remove.
+	AllowSplitUserKeys
+)
+
 // CheckOrdering checks that the files are consistent with respect to
 // seqnums (for level 0 files -- see detailed comment below) and increasing and non-
 // overlapping internal key ranges (for non-level 0 files).
-func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelIterator) error {
+//
+// The ordering field may be passed AllowSplitUserKeys to allow adjacent files that are both
+// inclusive of the same user key. Pebble no longer creates version edits
+// installing such files, and Pebble databases with sufficiently high format
+// major version should no longer have any such files within their LSM.
+// TODO(jackson): Remove AllowSplitUserKeys when we remove support for the
+// earlier format major versions.
+func CheckOrdering(
+	cmp Compare, format base.FormatKey, level Level, files LevelIterator, ordering OrderingInvariants,
+) error {
 	// The invariants to check for L0 sublevels are the same as the ones to
 	// check for all other levels. However, if L0 is not organized into
 	// sublevels, or if all L0 files are being passed in, we do the legacy L0
@@ -1527,11 +1540,29 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelI
 						prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
 						f.Smallest.Pretty(format), f.Largest.Pretty(format))
 				}
-				if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
+
-					return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
+				// What's considered "overlapping" is dependent on the format
-						errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
+				// major version. If ordering=ProhibitSplitUserKeys, then both
-						prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
+				// files cannot contain keys with the same user keys. If the
-						f.Smallest.Pretty(format), f.Largest.Pretty(format))
+				// bounds have the same user key, the previous file's boundary
+				// must have a Trailer indicating that it's exclusive.
+				switch ordering {
+				case AllowSplitUserKeys:
+					if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
+						return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
+							errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
+							prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
+							f.Smallest.Pretty(format), f.Largest.Pretty(format))
+					}
+				case ProhibitSplitUserKeys:
+					if v := cmp(prev.Largest.UserKey, f.Smallest.UserKey); v > 0 || (v == 0 && !prev.Largest.IsExclusiveSentinel()) {
+						return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]",
+							errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum),
+							prev.Smallest.Pretty(format), prev.Largest.Pretty(format),
+							f.Smallest.Pretty(format), f.Largest.Pretty(format))
+					}
+				default:
+					panic("unreachable")
 				}
 			}
 		}

vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go

@@ -783,6 +783,12 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error {
 		b.AddedFileBacking = make(map[base.DiskFileNum]*FileBacking)
 	}
 	for _, fb := range ve.CreatedBackingTables {
+		if _, ok := b.AddedFileBacking[fb.DiskFileNum]; ok {
+			// There is already a FileBacking associated with fb.DiskFileNum.
+			// This should never happen. There must always be only one FileBacking
+			// associated with a backing sstable.
+			panic(fmt.Sprintf("pebble: duplicate file backing %s", fb.DiskFileNum.String()))
+		}
 		b.AddedFileBacking[fb.DiskFileNum] = fb
 	}
 
@@ -846,6 +852,9 @@ func AccumulateIncompleteAndApplySingleVE(
 	flushSplitBytes int64,
 	readCompactionRate int64,
 	backingStateMap map[base.DiskFileNum]*FileBacking,
+	addBackingFunc func(*FileBacking),
+	removeBackingFunc func(base.DiskFileNum),
+	orderingInvariants OrderingInvariants,
 ) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) {
 	if len(ve.RemovedBackingTables) != 0 {
 		panic("pebble: invalid incomplete version edit")
@@ -857,14 +866,14 @@ func AccumulateIncompleteAndApplySingleVE(
 	}
 	zombies = make(map[base.DiskFileNum]uint64)
 	v, err := b.Apply(
-		curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies,
+		curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies, orderingInvariants,
 	)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	for _, s := range b.AddedFileBacking {
-		backingStateMap[s.DiskFileNum] = s
+		addBackingFunc(s)
 	}
 
 	for fileNum := range zombies {
@@ -875,10 +884,9 @@ func AccumulateIncompleteAndApplySingleVE(
 			ve.RemovedBackingTables = append(
 				ve.RemovedBackingTables, fileNum,
 			)
-			delete(backingStateMap, fileNum)
+			removeBackingFunc(fileNum)
 		}
 	}
-
 	return v, zombies, nil
 }
 
@@ -899,6 +907,7 @@ func (b *BulkVersionEdit) Apply(
 	flushSplitBytes int64,
 	readCompactionRate int64,
 	zombies map[base.DiskFileNum]uint64,
+	orderingInvariants OrderingInvariants,
 ) (*Version, error) {
 	addZombie := func(state *FileBacking) {
 		if zombies != nil {
@@ -1082,7 +1091,7 @@ func (b *BulkVersionEdit) Apply(
 			} else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil {
 				return nil, errors.Wrap(err, "pebble: internal error")
 			}
-			if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil {
+			if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter(), orderingInvariants); err != nil {
 				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 			continue
@@ -1103,7 +1112,7 @@ func (b *BulkVersionEdit) Apply(
 					end.Prev()
 				}
 			})
-			if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil {
+			if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter(), orderingInvariants); err != nil {
 				return nil, errors.Wrap(err, "pebble: internal error")
 			}
 		}

vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go

@@ -52,15 +52,15 @@ import (
 //	                        │
 //	                        ╰── <?>.Next
 type UserIteratorConfig struct {
-	snapshot   uint64
+	snapshot     uint64
-	comparer   *base.Comparer
+	comparer     *base.Comparer
-	miter      keyspan.MergingIter
+	miter        keyspan.MergingIter
-	biter      keyspan.BoundedIter
+	biter        keyspan.BoundedIter
-	diter      keyspan.DefragmentingIter
+	diter        keyspan.DefragmentingIter
-	liters     [manifest.NumLevels]keyspan.LevelIter
+	liters       [manifest.NumLevels]keyspan.LevelIter
-	litersUsed int
+	litersUsed   int
-	onlySets   bool
+	internalKeys bool
-	bufs       *Buffers
+	bufs         *Buffers
 }
 
 // Buffers holds various buffers used for range key iteration. They're exposed
@@ -79,9 +79,10 @@ func (bufs *Buffers) PrepareForReuse() {
 
 // Init initializes the range key iterator stack for user iteration. The
 // resulting fragment iterator applies range key semantics, defragments spans
-// according to their user-observable state and, if onlySets = true, removes all
+// according to their user-observable state and, if !internalKeys, removes all
 // Keys other than RangeKeySets describing the current state of range keys. The
-// resulting spans contain Keys sorted by Suffix.
+// resulting spans contain Keys sorted by suffix (unless internalKeys is true,
+// in which case they remain sorted by trailer descending).
 //
 // The snapshot sequence number parameter determines which keys are visible. Any
 // keys not visible at the provided snapshot are ignored.
@@ -91,16 +92,20 @@ func (ui *UserIteratorConfig) Init(
 	lower, upper []byte,
 	hasPrefix *bool,
 	prefix *[]byte,
-	onlySets bool,
+	internalKeys bool,
 	bufs *Buffers,
 	iters ...keyspan.FragmentIterator,
 ) keyspan.FragmentIterator {
 	ui.snapshot = snapshot
 	ui.comparer = comparer
-	ui.onlySets = onlySets
+	ui.internalKeys = internalKeys
 	ui.miter.Init(comparer.Compare, ui, &bufs.merging, iters...)
 	ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix)
-	ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	if internalKeys {
+		ui.diter.Init(comparer, &ui.biter, keyspan.DefragmentInternal, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	} else {
+		ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting)
+	}
 	ui.litersUsed = 0
 	ui.bufs = bufs
 	return &ui.diter
@@ -136,7 +141,8 @@ func (ui *UserIteratorConfig) SetBounds(lower, upper []byte) {
 // of unset keys, removal of keys overwritten by a set at the same suffix, etc)
 // and then non-RangeKeySet keys are removed. The resulting transformed spans
 // only contain RangeKeySets describing the state visible at the provided
-// sequence number, and hold their Keys sorted by Suffix.
+// sequence number, and hold their Keys sorted by Suffix (except if internalKeys
+// is true, then keys remain sorted by trailer.
 func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error {
 	// Apply shadowing of keys.
 	dst.Start = s.Start
@@ -148,9 +154,16 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 	if err := coalesce(ui.comparer.Equal, &ui.bufs.sortBuf, ui.snapshot, s.Keys); err != nil {
 		return err
 	}
-	// During user iteration over range keys, unsets and deletes don't matter.
+	if ui.internalKeys {
-	// Remove them if onlySets = true. This step helps logical defragmentation
+		if s.KeysOrder != keyspan.ByTrailerDesc {
-	// during iteration.
+			panic("unexpected key ordering in UserIteratorTransform with internalKeys = true")
+		}
+		dst.Keys = ui.bufs.sortBuf.Keys
+		keyspan.SortKeysByTrailer(&dst.Keys)
+		return nil
+	}
+	// During user iteration over range keys, unsets and deletes don't matter. This
+	// step helps logical defragmentation during iteration.
 	keys := ui.bufs.sortBuf.Keys
 	dst.Keys = dst.Keys[:0]
 	for i := range keys {
@@ -164,17 +177,11 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 			if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 {
 				panic("pebble: keys unexpectedly not in ascending suffix order")
 			}
-			if ui.onlySets {
+			// Skip.
-				// Skip.
+			continue
-				continue
-			}
-			dst.Keys = append(dst.Keys, keys[i])
 		case base.InternalKeyKindRangeKeyDelete:
-			if ui.onlySets {
+			// Skip.
-				// Skip.
+			continue
-				continue
-			}
-			dst.Keys = append(dst.Keys, keys[i])
 		default:
 			return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind())
 		}
@@ -193,6 +200,10 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k
 // sequence numbers). It's intended for use during user iteration, when the
 // wrapped keyspan iterator is merging spans across all levels of the LSM.
 func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.Span) bool {
+	// This method is not called with internalKeys = true.
+	if ui.internalKeys {
+		panic("unexpected call to ShouldDefragment with internalKeys = true")
+	}
 	// This implementation must only be used on spans that have transformed by
 	// ui.Transform. The transform applies shadowing, removes all keys besides
 	// the resulting Sets and sorts the keys by suffix. Since shadowing has been
@@ -208,8 +219,8 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S
 	ret := true
 	for i := range a.Keys {
 		if invariants.Enabled {
-			if ui.onlySets && (a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
+			if a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet ||
-				b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet) {
+				b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet {
 				panic("pebble: unexpected non-RangeKeySet during defragmentation")
 			}
 			if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 ||

vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go

@@ -384,14 +384,18 @@ func computeAlphabetKeyIndex(key []byte, alphabet map[byte]int64, n int) int64 {
 	return ret
 }
 
+func abs(a int64) int64 {
+	if a < 0 {
+		return -a
+	}
+	return a
+}
+
 // RandomSeparator returns a random alphabetic key k such that a < k < b,
 // pulling randomness from the provided random number generator. If dst is
 // provided and the generated key fits within dst's capacity, the returned slice
 // will use dst's memory.
 //
-// The keys a and b must be purely alphabetic, consisting only of characters
-// a-z.
-//
 // If a prefix P exists such that Prefix(a) < P < Prefix(b), the generated key
 // will consist of the prefix P appended with the provided suffix. A zero suffix
 // generates an unsuffixed key. If no such prefix P exists, RandomSeparator will
@@ -405,6 +409,10 @@ func computeAlphabetKeyIndex(key []byte, alphabet map[byte]int64, n int) int64 {
 //
 // RandomSeparator panics if a or b fails to decode.
 func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Rand) []byte {
+	if Comparer.Compare(a, b) >= 0 {
+		return nil
+	}
+
 	// Determine both keys' logical prefixes and suffixes.
 	ai := Comparer.Split(a)
 	bi := Comparer.Split(b)
@@ -431,31 +439,56 @@ func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Ra
 	diff := bpIdx - apIdx
 	generatedIdx := bpIdx
 	if diff > 0 {
+		var add int64 = diff + 1
+		var start int64 = apIdx
 		if as == 1 {
-			// There's no expressible key with suffix `a` greater than a@1, so
+			// There's no expressible key with prefix a greater than a@1. So,
 			// exclude ap.
-			generatedIdx = rng.Int63n(diff) + apIdx + 1
+			start = apIdx + 1
-		} else {
+			add = diff
-			generatedIdx = rng.Int63n(diff+1) + apIdx
+		}
+		if bs == 0 {
+			// No key with prefix b can sort before b@0. We don't want to pick b.
+			add--
+		}
+		// We're allowing generated id to be in the range [start, start + add - 1].
+		if start > start+add-1 {
+			return nil
+		}
+		// If we can generate a key which is actually in the middle of apIdx
+		// and bpIdx use it so that we don't have to bother about timestamps.
+		generatedIdx = rng.Int63n(add) + start
+		for diff > 1 && generatedIdx == apIdx || generatedIdx == bpIdx {
+			generatedIdx = rng.Int63n(add) + start
 		}
 	}
 
 	switch {
 	case generatedIdx == apIdx && generatedIdx == bpIdx:
-		if bs-as <= 1 {
+		if abs(bs-as) <= 1 {
 			// There's no expressible suffix between the two, and there's no
 			// possible separator key.
 			return nil
 		}
+		// The key b is >= key a, but has the same prefix, so b must have the
+		// smaller timestamp, unless a has timestamp of 0.
+		//
 		// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
 		// any suffix we generate will be greater than it.
-		if (as != 0 && suffix >= as) || suffix <= bs {
+		if as == 0 {
-			suffix = rng.Int63n(bs-as-1) + as + 1
+			// bs > as
+			suffix = bs + rng.Int63n(10) + 1
+		} else {
+			// bs < as.
+			// Generate suffix in range [bs + 1, as - 1]
+			suffix = bs + 1 + rng.Int63n(as-bs-1)
 		}
 	case generatedIdx == apIdx:
 		// NB: The zero suffix (suffix-less) sorts before all other suffixes, so
 		// any suffix we generate will be greater than it.
-		if as != 0 && suffix >= as {
+		if as == 0 && suffix == 0 {
+			suffix++
+		} else if as != 0 && suffix >= as {
 			suffix = rng.Int63n(as)
 		}
 	case generatedIdx == bpIdx:
@@ -468,7 +501,12 @@ func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Ra
 	} else {
 		dst = dst[:cap(dst)]
 	}
-	w := WriteKeyAt(dst, Alpha(maxLength), generatedIdx, suffix)
+	var w int
+	if suffix == 0 {
+		w = WriteKey(dst, Alpha(maxLength), generatedIdx)
+	} else {
+		w = WriteKeyAt(dst, Alpha(maxLength), generatedIdx, suffix)
+	}
 	return dst[:w]
 }
 

vendor/github.com/cockroachdb/pebble/iterator.go

@@ -447,6 +447,11 @@ const (
 	seekPrefixGELastPositioningOp
 	seekGELastPositioningOp
 	seekLTLastPositioningOp
+	// internalNextOp is a special internal iterator positioning operation used
+	// by CanDeterministicallySingleDelete. It exists for enforcing requirements
+	// around calling CanDeterministicallySingleDelete at most once per external
+	// iterator position.
+	internalNextOp
 	// invalidatedLastPositionOp is similar to unknownLastPositionOp and the
 	// only reason to distinguish this is for the wider set of SeekGE
 	// optimizations we permit for the external iterator Iterator.forwardOnly
@@ -550,6 +555,21 @@ func (i *Iterator) findNextEntry(limit []byte) {
 			return
 		}
 
+		// If the user has configured a SkipPoint function, invoke it to see
+		// whether we should skip over the current user key.
+		if i.opts.SkipPoint != nil && key.Kind() != InternalKeyKindRangeKeySet && i.opts.SkipPoint(i.iterKey.UserKey) {
+			// NB: We could call nextUserKey, but in some cases the SkipPoint
+			// predicate function might be cheaper than nextUserKey's key copy
+			// and key comparison. This should be the case for MVCC suffix
+			// comparisons, for example. In the future, we could expand the
+			// SkipPoint interface to give the implementor more control over
+			// whether we skip over just the internal key, the user key, or even
+			// the key prefix.
+			i.stats.ForwardStepCount[InternalIterCall]++
+			i.iterKey, i.iterValue = i.iter.Next()
+			continue
+		}
+
 		switch key.Kind() {
 		case InternalKeyKindRangeKeySet:
 			// Save the current key.
@@ -619,6 +639,13 @@ func (i *Iterator) findNextEntry(limit []byte) {
 }
 
 func (i *Iterator) nextPointCurrentUserKey() bool {
+	// If the user has configured a SkipPoint function and the current user key
+	// would be skipped by it, there's no need to step forward looking for a
+	// point key. If we were to find one, it should be skipped anyways.
+	if i.opts.SkipPoint != nil && i.opts.SkipPoint(i.key) {
+		return false
+	}
+
 	i.pos = iterPosCurForward
 
 	i.iterKey, i.iterValue = i.iter.Next()
@@ -780,39 +807,41 @@ func (i *Iterator) maybeSampleRead() {
 func (i *Iterator) sampleRead() {
 	var topFile *manifest.FileMetadata
 	topLevel, numOverlappingLevels := numLevels, 0
-	if mi, ok := i.iter.(*mergingIter); ok {
+	mi := i.merging
-		if len(mi.levels) > 1 {
+	if mi == nil {
-			mi.ForEachLevelIter(func(li *levelIter) bool {
+		return
-				l := manifest.LevelToInt(li.level)
+	}
-				if f := li.iterFile; f != nil {
+	if len(mi.levels) > 1 {
-					var containsKey bool
+		mi.ForEachLevelIter(func(li *levelIter) bool {
-					if i.pos == iterPosNext || i.pos == iterPosCurForward ||
+			l := manifest.LevelToInt(li.level)
-						i.pos == iterPosCurForwardPaused {
+			if f := li.iterFile; f != nil {
-						containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0
+				var containsKey bool
-					} else if i.pos == iterPosPrev || i.pos == iterPosCurReverse ||
+				if i.pos == iterPosNext || i.pos == iterPosCurForward ||
-						i.pos == iterPosCurReversePaused {
+					i.pos == iterPosCurForwardPaused {
-						containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0
+					containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0
-					}
+				} else if i.pos == iterPosPrev || i.pos == iterPosCurReverse ||
-					// Do nothing if the current key is not contained in f's
+					i.pos == iterPosCurReversePaused {
-					// bounds. We could seek the LevelIterator at this level
+					containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0
-					// to find the right file, but the performance impacts of
-					// doing that are significant enough to negate the benefits
-					// of read sampling in the first place. See the discussion
-					// at:
-					// https://github.com/cockroachdb/pebble/pull/1041#issuecomment-763226492
-					if containsKey {
-						numOverlappingLevels++
-						if numOverlappingLevels >= 2 {
-							// Terminate the loop early if at least 2 overlapping levels are found.
-							return true
-						}
-						topLevel = l
-						topFile = f
-					}
 				}
-				return false
+				// Do nothing if the current key is not contained in f's
-			})
+				// bounds. We could seek the LevelIterator at this level
-		}
+				// to find the right file, but the performance impacts of
+				// doing that are significant enough to negate the benefits
+				// of read sampling in the first place. See the discussion
+				// at:
+				// https://github.com/cockroachdb/pebble/pull/1041#issuecomment-763226492
+				if containsKey {
+					numOverlappingLevels++
+					if numOverlappingLevels >= 2 {
+						// Terminate the loop early if at least 2 overlapping levels are found.
+						return true
+					}
+					topLevel = l
+					topFile = f
+				}
+			}
+			return false
+		})
 	}
 	if topFile == nil || topLevel >= numLevels {
 		return
@@ -909,6 +938,26 @@ func (i *Iterator) findPrevEntry(limit []byte) {
 			}
 		}
 
+		// If the user has configured a SkipPoint function, invoke it to see
+		// whether we should skip over the current user key.
+		if i.opts.SkipPoint != nil && key.Kind() != InternalKeyKindRangeKeySet && i.opts.SkipPoint(key.UserKey) {
+			// NB: We could call prevUserKey, but in some cases the SkipPoint
+			// predicate function might be cheaper than prevUserKey's key copy
+			// and key comparison. This should be the case for MVCC suffix
+			// comparisons, for example. In the future, we could expand the
+			// SkipPoint interface to give the implementor more control over
+			// whether we skip over just the internal key, the user key, or even
+			// the key prefix.
+			i.stats.ReverseStepCount[InternalIterCall]++
+			i.iterKey, i.iterValue = i.iter.Prev()
+			if limit != nil && i.iterKey != nil && i.cmp(limit, i.iterKey.UserKey) > 0 && !i.rangeKeyWithinLimit(limit) {
+				i.iterValidityState = IterAtLimit
+				i.pos = iterPosCurReversePaused
+				return
+			}
+			continue
+		}
+
 		switch key.Kind() {
 		case InternalKeyKindRangeKeySet:
 			// Range key start boundary markers are interleaved with the maximum
@@ -946,12 +995,12 @@ func (i *Iterator) findPrevEntry(limit []byte) {
 			// Compare with the limit. We could optimize by only checking when
 			// we step to the previous user key, but detecting that requires a
 			// comparison too. Note that this position may already passed a
-			// number of versions of this user key, but they are all deleted,
+			// number of versions of this user key, but they are all deleted, so
-			// so the fact that a subsequent Prev*() call will not see them is
+			// the fact that a subsequent Prev*() call will not see them is
 			// harmless. Also note that this is the only place in the loop,
-			// other than the firstLoopIter case above, where we could step
+			// other than the firstLoopIter and SkipPoint cases above, where we
-			// to a different user key and start processing it for returning
+			// could step to a different user key and start processing it for
-			// to the caller.
+			// returning to the caller.
 			if limit != nil && i.iterKey != nil && i.cmp(limit, i.iterKey.UserKey) > 0 && !i.rangeKeyWithinLimit(limit) {
 				i.iterValidityState = IterAtLimit
 				i.pos = iterPosCurReversePaused
@@ -2132,6 +2181,9 @@ func (i *Iterator) Value() []byte {
 
 // ValueAndErr returns the value, and any error encountered in extracting the value.
 // REQUIRES: i.Error()==nil and HasPointAndRange() returns true for hasPoint.
+//
+// The caller should not modify the contents of the returned slice, and its
+// contents may change on the next call to Next.
 func (i *Iterator) ValueAndErr() ([]byte, error) {
 	val, callerOwned, err := i.value.Value(i.lazyValueBuf)
 	if err != nil {
@@ -2162,7 +2214,13 @@ func (i *Iterator) RangeKeys() []RangeKeyData {
 // Valid returns true if the iterator is positioned at a valid key/value pair
 // and false otherwise.
 func (i *Iterator) Valid() bool {
-	return i.iterValidityState == IterValid && !i.requiresReposition
+	valid := i.iterValidityState == IterValid && !i.requiresReposition
+	if invariants.Enabled {
+		if err := i.Error(); valid && err != nil {
+			panic(errors.WithSecondaryError(errors.AssertionFailedf("pebble: iterator is valid with non-nil Error"), err))
+		}
+	}
+	return valid
 }
 
 // Error returns any accumulated error.
@@ -2423,7 +2481,8 @@ func (i *Iterator) SetOptions(o *IterOptions) {
 	// If either options specify block property filters for an iterator stack,
 	// reconstruct it.
 	if i.pointIter != nil && (closeBoth || len(o.PointKeyFilters) > 0 || len(i.opts.PointKeyFilters) > 0 ||
-		o.RangeKeyMasking.Filter != nil || i.opts.RangeKeyMasking.Filter != nil) {
+		o.RangeKeyMasking.Filter != nil || i.opts.RangeKeyMasking.Filter != nil || o.SkipPoint != nil ||
+		i.opts.SkipPoint != nil) {
 		i.err = firstError(i.err, i.pointIter.Close())
 		i.pointIter = nil
 	}
@@ -2769,3 +2828,169 @@ func (stats *IteratorStats) SafeFormat(s redact.SafePrinter, verb rune) {
 			stats.RangeKeyStats.SkippedPoints)
 	}
 }
+
+// CanDeterministicallySingleDelete takes a valid iterator and examines internal
+// state to determine if a SingleDelete deleting Iterator.Key() would
+// deterministically delete the key. CanDeterministicallySingleDelete requires
+// the iterator to be oriented in the forward direction (eg, the last
+// positioning operation must've been a First, a Seek[Prefix]GE, or a
+// Next[Prefix][WithLimit]).
+//
+// This function does not change the external position of the iterator, and all
+// positioning methods should behave the same as if it was never called. This
+// function will only return a meaningful result the first time it's invoked at
+// an iterator position. This function invalidates the iterator Value's memory,
+// and the caller must not rely on the memory safety of the previous Iterator
+// position.
+//
+// If CanDeterministicallySingleDelete returns true AND the key at the iterator
+// position is not modified between the creation of the Iterator and the commit
+// of a batch containing a SingleDelete over the key, then the caller can be
+// assured that SingleDelete is equivalent to Delete on the local engine, but it
+// may not be true on another engine that received the same writes and with
+// logically equivalent state since this engine may have collapsed multiple SETs
+// into one.
+func CanDeterministicallySingleDelete(it *Iterator) (bool, error) {
+	// This function may only be called once per external iterator position. We
+	// can validate this by checking the last positioning operation.
+	if it.lastPositioningOp == internalNextOp {
+		return false, errors.New("pebble: CanDeterministicallySingleDelete called twice")
+	}
+	validity, kind := it.internalNext()
+	var shadowedBySingleDelete bool
+	for validity == internalNextValid {
+		switch kind {
+		case InternalKeyKindDelete, InternalKeyKindDeleteSized:
+			// A DEL or DELSIZED tombstone is okay. An internal key
+			// sequence like SINGLEDEL; SET; DEL; SET can be handled
+			// deterministically. If there are SETs further down, we
+			// don't care about them.
+			return true, nil
+		case InternalKeyKindSingleDelete:
+			// A SingleDelete is okay as long as when that SingleDelete was
+			// written, it was written deterministically (eg, with its own
+			// CanDeterministicallySingleDelete check). Validate that it was
+			// written deterministically. We'll allow one set to appear after
+			// the SingleDelete.
+			shadowedBySingleDelete = true
+			validity, kind = it.internalNext()
+			continue
+		case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge:
+			// If we observed a single delete, it's allowed to delete 1 key.
+			// We'll keep looping to validate that the internal keys beneath the
+			// already-written single delete are copacetic.
+			if shadowedBySingleDelete {
+				shadowedBySingleDelete = false
+				validity, kind = it.internalNext()
+				continue
+			}
+			// We encountered a shadowed SET, SETWITHDEL, MERGE. A SINGLEDEL
+			// that deleted the KV at the original iterator position could
+			// result in this key becoming visible.
+			return false, nil
+		case InternalKeyKindRangeDelete:
+			// RangeDeletes are handled by the merging iterator and should never
+			// be observed by the top-level Iterator.
+			panic(errors.AssertionFailedf("pebble: unexpected range delete"))
+		case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete:
+			// Range keys are interleaved at the maximal sequence number and
+			// should never be observed within a user key.
+			panic(errors.AssertionFailedf("pebble: unexpected range key"))
+		default:
+			panic(errors.AssertionFailedf("pebble: unexpected key kind: %s", errors.Safe(kind)))
+		}
+	}
+	if validity == internalNextError {
+		return false, it.Error()
+	}
+	return true, nil
+}
+
+// internalNextValidity enumerates the potential outcomes of a call to
+// internalNext.
+type internalNextValidity int8
+
+const (
+	// internalNextError is returned by internalNext when an error occurred and
+	// the caller is responsible for checking iter.Error().
+	internalNextError internalNextValidity = iota
+	// internalNextExhausted is returned by internalNext when the next internal
+	// key is an internal key with a different user key than Iterator.Key().
+	internalNextExhausted
+	// internalNextValid is returned by internalNext when the internal next
+	// found a shadowed internal key with a user key equal to Iterator.Key().
+	internalNextValid
+)
+
+// internalNext advances internal Iterator state forward to expose the
+// InternalKeyKind of the next internal key with a user key equal to Key().
+//
+// internalNext is a highly specialized operation and is unlikely to be
+// generally useful. See Iterator.Next for how to reposition the iterator to the
+// next key. internalNext requires the Iterator to be at a valid position in the
+// forward direction (the last positioning operation must've been a First, a
+// Seek[Prefix]GE, or a Next[Prefix][WithLimit] and Valid() must return true).
+//
+// internalNext, unlike all other Iterator methods, exposes internal LSM state.
+// internalNext advances the Iterator's internal iterator to the next shadowed
+// key with a user key equal to Key(). When a key is overwritten or deleted, its
+// removal from the LSM occurs lazily as a part of compactions. internalNext
+// allows the caller to see whether an obsolete internal key exists with the
+// current Key(), and what it's key kind is. Note that the existence of an
+// internal key is nondeterministic and dependent on internal LSM state. These
+// semantics are unlikely to be applicable to almost all use cases.
+//
+// If internalNext finds a key that shares the same user key as Key(), it
+// returns internalNextValid and the internal key's kind. If internalNext
+// encounters an error, it returns internalNextError and the caller is expected
+// to call Iterator.Error() to retrieve it. In all other circumstances,
+// internalNext returns internalNextExhausted, indicating that there are no more
+// additional internal keys with the user key Key().
+//
+// internalNext does not change the external position of the iterator, and a
+// Next operation should behave the same as if internalNext was never called.
+// internalNext does invalidate the iterator Value's memory, and the caller must
+// not rely on the memory safety of the previous Iterator position.
+func (i *Iterator) internalNext() (internalNextValidity, base.InternalKeyKind) {
+	i.stats.ForwardStepCount[InterfaceCall]++
+	if i.err != nil {
+		return internalNextError, base.InternalKeyKindInvalid
+	} else if i.iterValidityState != IterValid {
+		return internalNextExhausted, base.InternalKeyKindInvalid
+	}
+	i.lastPositioningOp = internalNextOp
+
+	switch i.pos {
+	case iterPosCurForward:
+		i.iterKey, i.iterValue = i.iter.Next()
+		if i.iterKey == nil {
+			// We check i.iter.Error() here and return an internalNextError enum
+			// variant so that the caller does not need to check i.iter.Error()
+			// in the common case that the next internal key has a new user key.
+			if i.err = i.iter.Error(); i.err != nil {
+				return internalNextError, base.InternalKeyKindInvalid
+			}
+			i.pos = iterPosNext
+			return internalNextExhausted, base.InternalKeyKindInvalid
+		} else if i.comparer.Equal(i.iterKey.UserKey, i.key) {
+			return internalNextValid, i.iterKey.Kind()
+		}
+		i.pos = iterPosNext
+		return internalNextExhausted, base.InternalKeyKindInvalid
+	case iterPosCurReverse, iterPosCurReversePaused, iterPosPrev:
+		i.err = errors.New("switching from reverse to forward via internalNext is prohibited")
+		i.iterValidityState = IterExhausted
+		return internalNextError, base.InternalKeyKindInvalid
+	case iterPosNext, iterPosCurForwardPaused:
+		// The previous method already moved onto the next user key. This is
+		// only possible if
+		//   - the last positioning method was a call to internalNext, and we
+		//     advanced to a new user key.
+		//   - the previous non-internalNext iterator operation encountered a
+		//     range key or merge, forcing an internal Next that found a new
+		//     user key that's not equal to i.Iterator.Key().
+		return internalNextExhausted, base.InternalKeyKindInvalid
+	default:
+		panic("unreachable")
+	}
+}

vendor/github.com/cockroachdb/pebble/level_checker.go

@@ -345,7 +345,7 @@ func iterateAndCheckTombstones(
 
 type checkConfig struct {
 	logger    Logger
-	cmp       Compare
+	comparer  *Comparer
 	readState *readState
 	newIters  tableNewIters
 	seqNum    uint64
@@ -354,6 +354,9 @@ type checkConfig struct {
 	formatKey base.FormatKey
 }
 
+// cmp is shorthand for comparer.Compare.
+func (c *checkConfig) cmp(a, b []byte) int { return c.comparer.Compare(a, b) }
+
 func checkRangeTombstones(c *checkConfig) error {
 	var level int
 	var tombstones []tombstoneWithLevel
@@ -571,7 +574,7 @@ func (d *DB) CheckLevels(stats *CheckLevelsStats) error {
 
 	checkConfig := &checkConfig{
 		logger:    d.opts.Logger,
-		cmp:       d.cmp,
+		comparer:  d.opts.Comparer,
 		readState: readState,
 		newIters:  d.newIters,
 		seqNum:    seqNum,
@@ -639,7 +642,7 @@ func checkLevelsInternal(c *checkConfig) (err error) {
 		manifestIter := current.L0SublevelFiles[sublevel].Iter()
 		iterOpts := IterOptions{logger: c.logger}
 		li := &levelIter{}
-		li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters, manifestIter,
+		li.init(context.Background(), iterOpts, c.comparer, c.newIters, manifestIter,
 			manifest.L0Sublevel(sublevel), internalIterOpts{})
 		li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
 		li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext)
@@ -653,7 +656,7 @@ func checkLevelsInternal(c *checkConfig) (err error) {
 
 		iterOpts := IterOptions{logger: c.logger}
 		li := &levelIter{}
-		li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters,
+		li.init(context.Background(), iterOpts, c.comparer, c.newIters,
 			current.Levels[level].Iter(), manifest.Level(level), internalIterOpts{})
 		li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
 		li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext)

vendor/github.com/cockroachdb/pebble/level_iter.go

@@ -82,10 +82,11 @@ type levelIter struct {
 	// short-lived (since they pin sstables), (b) plumbing a context into every
 	// method is very painful, (c) they do not (yet) respect context
 	// cancellation and are only used for tracing.
-	ctx    context.Context
+	ctx      context.Context
-	logger Logger
+	logger   Logger
-	cmp    Compare
+	comparer *Comparer
-	split  Split
+	cmp      Compare
+	split    Split
 	// The lower/upper bounds for iteration as specified at creation or the most
 	// recent call to SetBounds.
 	lower []byte
@@ -241,15 +242,14 @@ var _ base.InternalIterator = (*levelIter)(nil)
 // parameter if the caller is never going to call SeekPrefixGE.
 func newLevelIter(
 	opts IterOptions,
-	cmp Compare,
+	comparer *Comparer,
-	split Split,
 	newIters tableNewIters,
 	files manifest.LevelIterator,
 	level manifest.Level,
 	internalOpts internalIterOpts,
 ) *levelIter {
 	l := &levelIter{}
-	l.init(context.Background(), opts, cmp, split, newIters, files, level,
+	l.init(context.Background(), opts, comparer, newIters, files, level,
 		internalOpts)
 	return l
 }
@@ -257,8 +257,7 @@ func newLevelIter(
 func (l *levelIter) init(
 	ctx context.Context,
 	opts IterOptions,
-	cmp Compare,
+	comparer *Comparer,
-	split Split,
 	newIters tableNewIters,
 	files manifest.LevelIterator,
 	level manifest.Level,
@@ -278,8 +277,9 @@ func (l *levelIter) init(
 	l.tableOpts.UseL6Filters = opts.UseL6Filters
 	l.tableOpts.level = l.level
 	l.tableOpts.snapshotForHideObsoletePoints = opts.snapshotForHideObsoletePoints
-	l.cmp = cmp
+	l.comparer = comparer
-	l.split = split
+	l.cmp = comparer.Compare
+	l.split = comparer.Split
 	l.iterFile = nil
 	l.newIters = newIters
 	l.files = files

vendor/github.com/cockroachdb/pebble/mem_table.go

@@ -211,11 +211,13 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error {
 	var tombstoneCount, rangeKeyCount uint32
 	startSeqNum := seqNum
 	for r := batch.Reader(); ; seqNum++ {
-		kind, ukey, value, ok := r.Next()
+		kind, ukey, value, ok, err := r.Next()
 		if !ok {
+			if err != nil {
+				return err
+			}
 			break
 		}
-		var err error
 		ikey := base.MakeInternalKey(ukey, seqNum, kind)
 		switch kind {
 		case InternalKeyKindRangeDelete:

vendor/github.com/cockroachdb/pebble/merging_iter.go

@@ -27,6 +27,10 @@ type mergingIterLevel struct {
 	// iterKey and iterValue cache the current key and value iter are pointed at.
 	iterKey   *InternalKey
 	iterValue base.LazyValue
+	// levelIter is non-nil if this level's iter is ultimately backed by a
+	// *levelIter. The handle in iter may have wrapped the levelIter with
+	// intermediary internalIterator implementations.
+	levelIter *levelIter
 
 	// levelIterBoundaryContext's fields are set when using levelIter, in order
 	// to surface sstable boundary keys and file-level context. See levelIter
@@ -1371,9 +1375,9 @@ func (m *mergingIter) DebugString() string {
 }
 
 func (m *mergingIter) ForEachLevelIter(fn func(li *levelIter) bool) {
-	for _, iter := range m.levels {
+	for _, ml := range m.levels {
-		if li, ok := iter.iter.(*levelIter); ok {
+		if ml.levelIter != nil {
-			if done := fn(li); done {
+			if done := fn(ml.levelIter); done {
 				break
 			}
 		}

vendor/github.com/cockroachdb/pebble/metrics.go

@@ -44,9 +44,14 @@ type LevelMetrics struct {
 	Sublevels int32
 	// The total number of files in the level.
 	NumFiles int64
+	// The total number of virtual sstables in the level.
+	NumVirtualFiles uint64
 	// The total size in bytes of the files in the level.
 	Size int64
-	// The level's compaction score.
+	// The total size of the virtual sstables in the level.
+	VirtualSize uint64
+	// The level's compaction score. This is the compensatedScoreRatio in the
+	// candidateLevelInfo.
 	Score float64
 	// The number of incoming bytes from other levels read during
 	// compactions. This excludes bytes moved and bytes ingested. For L0 this is
@@ -77,6 +82,18 @@ type LevelMetrics struct {
 	TablesIngested uint64
 	// The number of sstables moved to this level by a "move" compaction.
 	TablesMoved uint64
+
+	MultiLevel struct {
+		// BytesInTop are the total bytes in a multilevel compaction coming from the top level.
+		BytesInTop uint64
+
+		// BytesIn, exclusively for multiLevel compactions.
+		BytesIn uint64
+
+		// BytesRead, exclusively for multilevel compactions.
+		BytesRead uint64
+	}
+
 	// Additional contains misc additional metrics that are not always printed.
 	Additional struct {
 		// The sum of Properties.ValueBlocksSize for all the sstables in this
@@ -94,6 +111,8 @@ type LevelMetrics struct {
 // Add updates the counter metrics for the level.
 func (m *LevelMetrics) Add(u *LevelMetrics) {
 	m.NumFiles += u.NumFiles
+	m.NumVirtualFiles += u.NumVirtualFiles
+	m.VirtualSize += u.VirtualSize
 	m.Size += u.Size
 	m.BytesIn += u.BytesIn
 	m.BytesIngested += u.BytesIngested
@@ -105,6 +124,9 @@ func (m *LevelMetrics) Add(u *LevelMetrics) {
 	m.TablesFlushed += u.TablesFlushed
 	m.TablesIngested += u.TablesIngested
 	m.TablesMoved += u.TablesMoved
+	m.MultiLevel.BytesInTop += u.MultiLevel.BytesInTop
+	m.MultiLevel.BytesRead += u.MultiLevel.BytesRead
+	m.MultiLevel.BytesIn += u.MultiLevel.BytesIn
 	m.Additional.BytesWrittenDataBlocks += u.Additional.BytesWrittenDataBlocks
 	m.Additional.BytesWrittenValueBlocks += u.Additional.BytesWrittenValueBlocks
 	m.Additional.ValueBlocksSize += u.Additional.ValueBlocksSize
@@ -130,14 +152,15 @@ type Metrics struct {
 
 	Compact struct {
 		// The total number of compactions, and per-compaction type counts.
-		Count            int64
+		Count             int64
-		DefaultCount     int64
+		DefaultCount      int64
-		DeleteOnlyCount  int64
+		DeleteOnlyCount   int64
-		ElisionOnlyCount int64
+		ElisionOnlyCount  int64
-		MoveCount        int64
+		MoveCount         int64
-		ReadCount        int64
+		ReadCount         int64
-		RewriteCount     int64
+		RewriteCount      int64
-		MultiLevelCount  int64
+		MultiLevelCount   int64
+		CounterLevelCount int64
 		// An estimate of the number of bytes that need to be compacted for the LSM
 		// to reach a stable state.
 		EstimatedDebt uint64
@@ -236,6 +259,10 @@ type Metrics struct {
 		ZombieSize uint64
 		// The count of zombie tables.
 		ZombieCount int64
+		// The count of the backing sstables.
+		BackingTableCount uint64
+		// The sum of the sizes of the all of the backing sstables.
+		BackingTableSize uint64
 	}
 
 	TableCache CacheMetrics
@@ -284,6 +311,13 @@ var (
 		prometheus.LinearBuckets(0.0, float64(time.Microsecond*100), 50),
 		prometheus.ExponentialBucketsRange(float64(time.Millisecond*5), float64(10*time.Second), 50)...,
 	)
+
+	// SecondaryCacheIOBuckets exported to enable exporting from package pebble to
+	// enable exporting metrics with below buckets in CRDB.
+	SecondaryCacheIOBuckets = sharedcache.IOBuckets
+	// SecondaryCacheChannelWriteBuckets exported to enable exporting from package
+	// pebble to enable exporting metrics with below buckets in CRDB.
+	SecondaryCacheChannelWriteBuckets = sharedcache.ChannelWriteBuckets
 )
 
 // DiskSpaceUsage returns the total disk space used by the database in bytes,
@@ -303,6 +337,27 @@ func (m *Metrics) DiskSpaceUsage() uint64 {
 	return usageBytes
 }
 
+// NumVirtual is the number of virtual sstables in the latest version
+// summed over every level in the lsm.
+func (m *Metrics) NumVirtual() uint64 {
+	var n uint64
+	for _, level := range m.Levels {
+		n += level.NumVirtualFiles
+	}
+	return n
+}
+
+// VirtualSize is the sum of the sizes of the virtual sstables in the
+// latest version. BackingTableSize - VirtualSize gives an estimate for
+// the space amplification caused by not compacting virtual sstables.
+func (m *Metrics) VirtualSize() uint64 {
+	var size uint64
+	for _, level := range m.Levels {
+		size += level.VirtualSize
+	}
+	return size
+}
+
 // ReadAmp returns the current read amplification of the database.
 // It's computed as the number of sublevels in L0 + the number of non-empty
 // levels below L0.
@@ -333,24 +388,25 @@ func (m *Metrics) Total() LevelMetrics {
 
 // String pretty-prints the metrics as below:
 //
-//	      |                     |       |       |   ingested   |     moved    |    written   |       |    amp
+//	      |                             |       |       |   ingested   |     moved    |    written   |       |    amp
-//	level | tables  size val-bl | score |   in  | tables  size | tables  size | tables  size |  read |   r   w
+//	level | tables  size val-bl vtables | score |   in  | tables  size | tables  size | tables  size |  read |   r   w
-//	------+---------------------+-------+-------+--------------+--------------+--------------+-------+---------
+//	------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+---------
-//	    0 |   101   102B     0B | 103.0 |  104B |   112   104B |   113   106B |   221   217B |  107B |   1  2.1
+//	    0 |   101   102B     0B       0 | 103.0 |  104B |   112   104B |   113   106B |   221   217B |  107B |   1  2.1
-//	    1 |   201   202B     0B | 203.0 |  204B |   212   204B |   213   206B |   421   417B |  207B |   2  2.0
+//	    1 |   201   202B     0B       0 | 203.0 |  204B |   212   204B |   213   206B |   421   417B |  207B |   2  2.0
-//	    2 |   301   302B     0B | 303.0 |  304B |   312   304B |   313   306B |   621   617B |  307B |   3  2.0
+//	    2 |   301   302B     0B       0 | 303.0 |  304B |   312   304B |   313   306B |   621   617B |  307B |   3  2.0
-//	    3 |   401   402B     0B | 403.0 |  404B |   412   404B |   413   406B |   821   817B |  407B |   4  2.0
+//	    3 |   401   402B     0B       0 | 403.0 |  404B |   412   404B |   413   406B |   821   817B |  407B |   4  2.0
-//	    4 |   501   502B     0B | 503.0 |  504B |   512   504B |   513   506B |  1.0K  1017B |  507B |   5  2.0
+//	    4 |   501   502B     0B       0 | 503.0 |  504B |   512   504B |   513   506B |  1.0K  1017B |  507B |   5  2.0
-//	    5 |   601   602B     0B | 603.0 |  604B |   612   604B |   613   606B |  1.2K  1.2KB |  607B |   6  2.0
+//	    5 |   601   602B     0B       0 | 603.0 |  604B |   612   604B |   613   606B |  1.2K  1.2KB |  607B |   6  2.0
-//	    6 |   701   702B     0B |     - |  704B |   712   704B |   713   706B |  1.4K  1.4KB |  707B |   7  2.0
+//	    6 |   701   702B     0B       0 |     - |  704B |   712   704B |   713   706B |  1.4K  1.4KB |  707B |   7  2.0
-//	total |  2.8K  2.7KB     0B |     - | 2.8KB |  2.9K  2.8KB |  2.9K  2.8KB |  5.7K  8.4KB | 2.8KB |  28  3.0
+//	total |  2.8K  2.7KB     0B       0 |     - | 2.8KB |  2.9K  2.8KB |  2.9K  2.8KB |  5.7K  8.4KB | 2.8KB |  28  3.0
-//	-----------------------------------------------------------------------------------------------------------
+//	-------------------------------------------------------------------------------------------------------------------
 //	WAL: 22 files (24B)  in: 25B  written: 26B (4% overhead)
 //	Flushes: 8
 //	Compactions: 5  estimated debt: 6B  in progress: 2 (7B)
 //	default: 27  delete: 28  elision: 29  move: 30  read: 31  rewrite: 32  multi-level: 33
 //	MemTables: 12 (11B)  zombie: 14 (13B)
 //	Zombie tables: 16 (15B)
+//	Backing tables: 0 (0B)
 //	Block cache: 2 entries (1B)  hit rate: 42.9%
 //	Table cache: 18 entries (17B)  hit rate: 48.7%
 //	Secondary cache: 40 entries (40B)  hit rate: 49.9%
@@ -378,9 +434,25 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 	// width specifiers. When the issue is fixed, we can convert these to
 	// RedactableStrings. https://github.com/cockroachdb/redact/issues/17
 
-	w.SafeString("      |                     |       |       |   ingested   |     moved    |    written   |       |    amp\n")
+	multiExists := m.Compact.MultiLevelCount > 0
-	w.SafeString("level | tables  size val-bl | score |   in  | tables  size | tables  size | tables  size |  read |   r   w\n")
+	appendIfMulti := func(line redact.SafeString) {
-	w.SafeString("------+---------------------+-------+-------+--------------+--------------+--------------+-------+---------\n")
+		if multiExists {
+			w.SafeString(line)
+		}
+	}
+	newline := func() {
+		w.SafeString("\n")
+	}
+
+	w.SafeString("      |                             |       |       |   ingested   |     moved    |    written   |       |    amp")
+	appendIfMulti("   |     multilevel")
+	newline()
+	w.SafeString("level | tables  size val-bl vtables | score |   in  | tables  size | tables  size | tables  size |  read |   r   w")
+	appendIfMulti("  |    top   in  read")
+	newline()
+	w.SafeString("------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+---------")
+	appendIfMulti("-+------------------")
+	newline()
 
 	// formatRow prints out a row of the table.
 	formatRow := func(m *LevelMetrics, score float64) {
@@ -403,10 +475,11 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 			wampStr = fmt.Sprintf("%.1f", wamp)
 		}
 
-		w.Printf("| %5s %6s %6s | %5s | %5s | %5s %6s | %5s %6s | %5s %6s | %5s | %3d %4s\n",
+		w.Printf("| %5s %6s %6s %7s | %5s | %5s | %5s %6s | %5s %6s | %5s %6s | %5s | %3d %4s",
 			humanize.Count.Int64(m.NumFiles),
 			humanize.Bytes.Int64(m.Size),
 			humanize.Bytes.Uint64(m.Additional.ValueBlocksSize),
+			humanize.Count.Uint64(m.NumVirtualFiles),
 			redact.Safe(scoreStr),
 			humanize.Bytes.Uint64(m.BytesIn),
 			humanize.Count.Uint64(m.TablesIngested),
@@ -418,6 +491,14 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 			humanize.Bytes.Uint64(m.BytesRead),
 			redact.Safe(m.Sublevels),
 			redact.Safe(wampStr))
+
+		if multiExists {
+			w.Printf(" | %5s %5s %5s",
+				humanize.Bytes.Uint64(m.MultiLevel.BytesInTop),
+				humanize.Bytes.Uint64(m.MultiLevel.BytesIn),
+				humanize.Bytes.Uint64(m.MultiLevel.BytesRead))
+		}
+		newline()
 	}
 
 	var total LevelMetrics
@@ -443,8 +524,9 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 	w.SafeString("total ")
 	formatRow(&total, math.NaN())
 
-	w.SafeString("-----------------------------------------------------------------------------------------------------------\n")
+	w.SafeString("-------------------------------------------------------------------------------------------------------------------")
-
+	appendIfMulti("--------------------")
+	newline()
 	w.Printf("WAL: %d files (%s)  in: %s  written: %s (%.0f%% overhead)\n",
 		redact.Safe(m.WAL.Files),
 		humanize.Bytes.Uint64(m.WAL.Size),
@@ -479,6 +561,13 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 		redact.Safe(m.Table.ZombieCount),
 		humanize.Bytes.Uint64(m.Table.ZombieSize))
 
+	w.Printf("Backing tables: %d (%s)\n",
+		redact.Safe(m.Table.BackingTableCount),
+		humanize.Bytes.Uint64(m.Table.BackingTableSize))
+	w.Printf("Virtual tables: %d (%s)\n",
+		redact.Safe(m.NumVirtual()),
+		humanize.Bytes.Uint64(m.VirtualSize()))
+
 	formatCacheMetrics := func(m *CacheMetrics, name redact.SafeString) {
 		w.Printf("%s: %s entries (%s)  hit rate: %.1f%%\n",
 			name,

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go

@@ -97,10 +97,10 @@ type Settings struct {
 	Remote struct {
 		StorageFactory remote.StorageFactory
 
-		// If CreateOnShared is true, sstables are created on remote storage using
+		// If CreateOnShared is non-zero, sstables are created on remote storage using
 		// the CreateOnSharedLocator (when the PreferSharedStorage create option is
 		// true).
-		CreateOnShared        bool
+		CreateOnShared        remote.CreateOnSharedStrategy
 		CreateOnSharedLocator remote.Locator
 
 		// CacheSizeBytes is the size of the on-disk block cache for objects
@@ -249,7 +249,7 @@ func (p *provider) Create(
 	fileNum base.DiskFileNum,
 	opts objstorage.CreateOptions,
 ) (w objstorage.Writable, meta objstorage.ObjectMetadata, err error) {
-	if opts.PreferSharedStorage && p.st.Remote.CreateOnShared {
+	if opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone {
 		w, meta, err = p.sharedCreate(ctx, fileType, fileNum, p.st.Remote.CreateOnSharedLocator, opts)
 	} else {
 		w, meta, err = p.vfsCreate(ctx, fileType, fileNum)
@@ -338,7 +338,7 @@ func (p *provider) LinkOrCopyFromLocal(
 	dstFileNum base.DiskFileNum,
 	opts objstorage.CreateOptions,
 ) (objstorage.ObjectMetadata, error) {
-	shared := opts.PreferSharedStorage && p.st.Remote.CreateOnShared
+	shared := opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone
 	if !shared && srcFS == p.st.FS {
 		// Wrap the normal filesystem with one which wraps newly created files with
 		// vfs.NewSyncingFile.

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/version_edit.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
+	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/objstorage/remote"
 )
@@ -234,14 +235,18 @@ func (v *VersionEdit) Apply(
 		*creatorID = v.CreatorID
 	}
 	for _, meta := range v.NewObjects {
-		if _, exists := objects[meta.FileNum]; exists {
+		if invariants.Enabled {
-			return errors.AssertionFailedf("version edit adds existing object %s", meta.FileNum)
+			if _, exists := objects[meta.FileNum]; exists {
+				return errors.AssertionFailedf("version edit adds existing object %s", meta.FileNum)
+			}
 		}
 		objects[meta.FileNum] = meta
 	}
 	for _, fileNum := range v.DeletedObjects {
-		if _, exists := objects[fileNum]; !exists {
+		if invariants.Enabled {
-			return errors.AssertionFailedf("version edit deletes non-existent object %s", fileNum)
+			if _, exists := objects[fileNum]; !exists {
+				return errors.AssertionFailedf("version edit deletes non-existent object %s", fileNum)
+			}
 		}
 		delete(objects, fileNum)
 	}

vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache/shared_cache.go

@@ -21,6 +21,13 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+// Exported to enable exporting from package pebble to enable
+// exporting metrics with below buckets in CRDB.
+var (
+	IOBuckets           = prometheus.ExponentialBucketsRange(float64(time.Millisecond*1), float64(10*time.Second), 50)
+	ChannelWriteBuckets = prometheus.ExponentialBucketsRange(float64(time.Microsecond*1), float64(10*time.Second), 50)
+)
+
 // Cache is a persistent cache backed by a local filesystem. It is intended
 // to cache data that is in slower shared storage (e.g. S3), hence the
 // package name 'sharedcache'.
@@ -123,9 +130,10 @@ func Open(
 	sizeBytes int64,
 	numShards int,
 ) (*Cache, error) {
-	min := shardingBlockSize * int64(numShards)
+	if minSize := shardingBlockSize * int64(numShards); sizeBytes < minSize {
-	if sizeBytes < min {
+		// Up the size so that we have one block per shard. In practice, this should
-		return nil, errors.Errorf("cache size %d lower than min %d", sizeBytes, min)
+		// only happen in tests.
+		sizeBytes = minSize
 	}
 
 	c := &Cache{
@@ -143,15 +151,13 @@ func Open(
 
 	c.writeWorkers.Start(c, numShards*writeWorkersPerShard)
 
-	buckets := prometheus.ExponentialBucketsRange(float64(time.Millisecond*1), float64(10*time.Second), 50)
+	c.metrics.getLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
-	c.metrics.getLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: buckets})
+	c.metrics.diskReadLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
-	c.metrics.diskReadLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: buckets})
+	c.metrics.putLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
-	c.metrics.putLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: buckets})
+	c.metrics.diskWriteLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets})
-	c.metrics.diskWriteLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: buckets})
 
 	// Measures a channel write, so lower min.
-	buckets = prometheus.ExponentialBucketsRange(float64(time.Microsecond*1), float64(10*time.Second), 50)
+	c.metrics.queuePutLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: ChannelWriteBuckets})
-	c.metrics.queuePutLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: buckets})
 
 	return c, nil
 }

vendor/github.com/cockroachdb/pebble/objstorage/remote/storage.go

@@ -22,6 +22,42 @@ type StorageFactory interface {
 	CreateStorage(locator Locator) (Storage, error)
 }
 
+// SharedLevelsStart denotes the highest (i.e. lowest numbered) level that will
+// have sstables shared across Pebble instances when doing skip-shared
+// iteration (see db.ScanInternal) or shared file ingestion (see
+// db.IngestAndExcise).
+const SharedLevelsStart = 5
+
+// CreateOnSharedStrategy specifies what table files should be created on shared
+// storage. For use with CreateOnShared in options.
+type CreateOnSharedStrategy int
+
+const (
+	// CreateOnSharedNone denotes no files being created on shared storage.
+	CreateOnSharedNone CreateOnSharedStrategy = iota
+	// CreateOnSharedLower denotes the creation of files in lower levels of the
+	// LSM (specifically, L5 and L6 as they're below SharedLevelsStart) on
+	// shared storage, and higher levels on local storage.
+	CreateOnSharedLower
+	// CreateOnSharedAll denotes the creation of all sstables on shared storage.
+	CreateOnSharedAll
+)
+
+// ShouldCreateShared returns whether new table files at the specified level
+// should be created on shared storage.
+func ShouldCreateShared(strategy CreateOnSharedStrategy, level int) bool {
+	switch strategy {
+	case CreateOnSharedAll:
+		return true
+	case CreateOnSharedNone:
+		return false
+	case CreateOnSharedLower:
+		return level >= SharedLevelsStart
+	default:
+		panic("unexpected CreateOnSharedStrategy value")
+	}
+}
+
 // Storage is an interface for a blob storage driver. This is lower-level
 // than an FS-like interface, however FS/File-like abstractions can be built on
 // top of these methods.

vendor/github.com/cockroachdb/pebble/open.go

@@ -89,7 +89,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 	// Open the database and WAL directories first.
 	walDirname, dataDir, walDir, err := prepareAndOpenDirs(dirname, opts)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "error opening database at %q", dirname)
 	}
 	defer func() {
 		if db == nil {
@@ -260,7 +260,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 		}
 
 		// Create the DB.
-		if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, setCurrent, &d.mu.Mutex); err != nil {
+		if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, setCurrent, d.FormatMajorVersion, &d.mu.Mutex); err != nil {
 			return nil, err
 		}
 	} else {
@@ -268,7 +268,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 			return nil, errors.Wrapf(ErrDBAlreadyExists, "dirname=%q", dirname)
 		}
 		// Load the version set.
-		if err := d.mu.versions.load(dirname, opts, manifestFileNum.FileNum(), manifestMarker, setCurrent, &d.mu.Mutex); err != nil {
+		if err := d.mu.versions.load(dirname, opts, manifestFileNum.FileNum(), manifestMarker, setCurrent, d.FormatMajorVersion, &d.mu.Mutex); err != nil {
 			return nil, err
 		}
 		if opts.ErrorIfNotPristine {
@@ -381,6 +381,15 @@ func Open(dirname string, opts *Options) (db *DB, _ error) {
 		}
 	}
 
+	// Ratchet d.mu.versions.nextFileNum ahead of all known objects in the
+	// objProvider. This avoids FileNum collisions with obsolete sstables.
+	objects := d.objProvider.List()
+	for _, obj := range objects {
+		if d.mu.versions.nextFileNum <= obj.DiskFileNum.FileNum() {
+			d.mu.versions.nextFileNum = obj.DiskFileNum.FileNum() + 1
+		}
+	}
+
 	// Validate the most-recent OPTIONS file, if there is one.
 	var strictWALTail bool
 	if previousOptionsFilename != "" {
@@ -700,6 +709,11 @@ func (d *DB) replayWAL(
 		batchesReplayed int64 // number of batches replayed
 	)
 
+	// TODO(jackson): This function is interspersed with panics, in addition to
+	// corruption error propagation. Audit them to ensure we're truly only
+	// panicking where the error points to Pebble bug and not user or
+	// hardware-induced corruption.
+
 	if d.opts.ReadOnly {
 		// In read-only mode, we replay directly into the mutable memtable which will
 		// never be flushed.
@@ -754,6 +768,11 @@ func (d *DB) replayWAL(
 		ve.NewFiles = append(ve.NewFiles, newVE.NewFiles...)
 		return nil
 	}
+	defer func() {
+		if err != nil {
+			err = errors.WithDetailf(err, "replaying log %s, offset %d", logNum, offset)
+		}
+	}()
 
 	for {
 		offset = rr.Offset()
@@ -786,7 +805,8 @@ func (d *DB) replayWAL(
 
 		// Specify Batch.db so that Batch.SetRepr will compute Batch.memTableSize
 		// which is used below.
-		b = Batch{db: d}
+		b = Batch{}
+		b.db = d
 		b.SetRepr(buf.Bytes())
 		seqNum := b.SeqNum()
 		maxSeqNum = seqNum + uint64(b.Count())
@@ -794,7 +814,9 @@ func (d *DB) replayWAL(
 		batchesReplayed++
 		{
 			br := b.Reader()
-			if kind, encodedFileNum, _, _ := br.Next(); kind == InternalKeyKindIngestSST {
+			if kind, encodedFileNum, _, ok, err := br.Next(); err != nil {
+				return nil, 0, err
+			} else if ok && kind == InternalKeyKindIngestSST {
 				fileNums := make([]base.DiskFileNum, 0, b.Count())
 				addFileNum := func(encodedFileNum []byte) {
 					fileNum, n := binary.Uvarint(encodedFileNum)
@@ -806,7 +828,10 @@ func (d *DB) replayWAL(
 				addFileNum(encodedFileNum)
 
 				for i := 1; i < int(b.Count()); i++ {
-					kind, encodedFileNum, _, ok := br.Next()
+					kind, encodedFileNum, _, ok, err := br.Next()
+					if err != nil {
+						return nil, 0, err
+					}
 					if kind != InternalKeyKindIngestSST {
 						panic("pebble: invalid batch key kind.")
 					}
@@ -816,7 +841,9 @@ func (d *DB) replayWAL(
 					addFileNum(encodedFileNum)
 				}
 
-				if _, _, _, ok := br.Next(); ok {
+				if _, _, _, ok, err := br.Next(); err != nil {
+					return nil, 0, err
+				} else if ok {
 					panic("pebble: invalid number of entries in batch.")
 				}
 
@@ -909,7 +936,10 @@ func (d *DB) replayWAL(
 			// Make a copy of the data slice since it is currently owned by buf and will
 			// be reused in the next iteration.
 			b.data = append([]byte(nil), b.data...)
-			b.flushable = newFlushableBatch(&b, d.opts.Comparer)
+			b.flushable, err = newFlushableBatch(&b, d.opts.Comparer)
+			if err != nil {
+				return nil, 0, err
+			}
 			entry := d.newFlushableEntry(b.flushable, logNum, b.SeqNum())
 			// Disable memory accounting by adding a reader ref that will never be
 			// removed.
@@ -1106,6 +1136,12 @@ var ErrDBAlreadyExists = errors.New("pebble: database already exists")
 // Note that errors can be wrapped with more details; use errors.Is().
 var ErrDBNotPristine = errors.New("pebble: database already exists and is not pristine")
 
+// IsCorruptionError returns true if the given error indicates database
+// corruption.
+func IsCorruptionError(err error) bool {
+	return errors.Is(err, base.ErrCorruption)
+}
+
 func checkConsistency(v *manifest.Version, dirname string, objProvider objstorage.Provider) error {
 	var buf bytes.Buffer
 	var args []interface{}

vendor/github.com/cockroachdb/pebble/options.go

@@ -119,6 +119,16 @@ type IterOptions struct {
 	// false to skip scanning. This function must be thread-safe since the same
 	// function can be used by multiple iterators, if the iterator is cloned.
 	TableFilter func(userProps map[string]string) bool
+	// SkipPoint may be used to skip over point keys that don't match an
+	// arbitrary predicate during iteration. If set, the Iterator invokes
+	// SkipPoint for keys encountered. If SkipPoint returns true, the iterator
+	// will skip the key without yielding it to the iterator operation in
+	// progress.
+	//
+	// SkipPoint must be a pure function and always return the same result when
+	// provided the same arguments. The iterator may call SkipPoint multiple
+	// times for the same user key.
+	SkipPoint func(userKey []byte) bool
 	// PointKeyFilters can be used to avoid scanning tables and blocks in tables
 	// when iterating over point keys. This slice represents an intersection
 	// across all filters, i.e., all filters must indicate that the block is
@@ -540,6 +550,11 @@ type Options struct {
 		// concurrency slots as determined by the two options is chosen.
 		CompactionDebtConcurrency uint64
 
+		// IngestSplit, if it returns true, allows for ingest-time splitting of
+		// existing sstables into two virtual sstables to allow ingestion sstables to
+		// slot into a lower level than they otherwise would have.
+		IngestSplit func() bool
+
 		// ReadCompactionRate controls the frequency of read triggered
 		// compactions by adjusting `AllowedSeeks` in manifest.FileMetadata:
 		//
@@ -603,10 +618,10 @@ type Options struct {
 		// desired size of each level of the LSM. Defaults to 10.
 		LevelMultiplier int
 
-		// MultiLevelCompactionHueristic determines whether to add an additional
+		// MultiLevelCompactionHeuristic determines whether to add an additional
 		// level to a conventional two level compaction. If nil, a multilevel
 		// compaction will never get triggered.
-		MultiLevelCompactionHueristic MultiLevelHeuristic
+		MultiLevelCompactionHeuristic MultiLevelHeuristic
 
 		// MaxWriterConcurrency is used to indicate the maximum number of
 		// compression workers the compression queue is allowed to use. If
@@ -670,19 +685,44 @@ type Options struct {
 		// allows ingestion of external files.
 		RemoteStorage remote.StorageFactory
 
-		// If CreateOnShared is true, any new sstables are created on remote storage
+		// If CreateOnShared is non-zero, new sstables are created on remote storage
-		// (using CreateOnSharedLocator). These sstables can be shared between
+		// (using CreateOnSharedLocator and with the appropriate
-		// different Pebble instances; the lifecycle of such objects is managed by
+		// CreateOnSharedStrategy). These sstables can be shared between different
-		// the cluster.
+		// Pebble instances; the lifecycle of such objects is managed by the
+		// remote.Storage constructed by options.RemoteStorage.
 		//
 		// Can only be used when RemoteStorage is set (and recognizes
 		// CreateOnSharedLocator).
-		CreateOnShared        bool
+		CreateOnShared        remote.CreateOnSharedStrategy
 		CreateOnSharedLocator remote.Locator
 
 		// CacheSizeBytesBytes is the size of the on-disk block cache for objects
 		// on shared storage in bytes. If it is 0, no cache is used.
 		SecondaryCacheSizeBytes int64
+
+		// IneffectualPointDeleteCallback is called in compactions/flushes if any
+		// single delete is being elided without deleting a point set/merge.
+		IneffectualSingleDeleteCallback func(userKey []byte)
+
+		// SingleDeleteInvariantViolationCallback is called in compactions/flushes if any
+		// single delete has consumed a Set/Merge, and there is another immediately older
+		// Set/SetWithDelete/Merge. The user of Pebble has violated the invariant under
+		// which SingleDelete can be used correctly.
+		//
+		// Consider the sequence SingleDelete#3, Set#2, Set#1. There are three
+		// ways some of these keys can first meet in a compaction.
+		//
+		// - All 3 keys in the same compaction: this callback will detect the
+		//   violation.
+		//
+		// - SingleDelete#3, Set#2 meet in a compaction first: Both keys will
+		//   disappear. The violation will not be detected, and the DB will have
+		//   Set#1 which is likely incorrect (from the user's perspective).
+		//
+		// - Set#2, Set#1 meet in a compaction first: The output will be Set#2,
+		//   which will later be consumed by SingleDelete#3. The violation will
+		//   not be detected and the DB will be correct.
+		SingleDeleteInvariantViolationCallback func(userKey []byte)
 	}
 
 	// Filters is a map from filter policy name to filter policy. It is used for
@@ -797,12 +837,18 @@ type Options struct {
 	// writing the contents of the old one in the
 	// background. MemTableStopWritesThreshold places a hard limit on the size of
 	// the queued MemTables.
+	//
+	// The default value is 4MB.
 	MemTableSize uint64
 
-	// Hard limit on the size of queued of MemTables. Writes are stopped when the
+	// Hard limit on the number of queued of MemTables. Writes are stopped when
-	// sum of the queued memtable sizes exceeds
+	// the sum of the queued memtable sizes exceeds:
-	// MemTableStopWritesThreshold*MemTableSize. This value should be at least 2
+	//   MemTableStopWritesThreshold * MemTableSize.
-	// or writes will stop whenever a MemTable is being flushed.
+	//
+	// This value should be at least 2 or writes will stop whenever a MemTable is
+	// being flushed.
+	//
+	// The default value is 2.
 	MemTableStopWritesThreshold int
 
 	// Merger defines the associative merge operation to use for merging values
@@ -934,6 +980,10 @@ type Options struct {
 		// A private option to disable stats collection.
 		disableTableStats bool
 
+		// testingAlwaysWaitForCleanup is set by some tests to force waiting for
+		// obsolete file deletion (to make events deterministic).
+		testingAlwaysWaitForCleanup bool
+
 		// fsCloser holds a closer that should be invoked after a DB using these
 		// Options is closed. This is used to automatically stop the
 		// long-running goroutine associated with the disk-health-checking FS.
@@ -1045,7 +1095,7 @@ func (o *Options) EnsureDefaults() *Options {
 		o.MaxOpenFiles = 1000
 	}
 	if o.MemTableSize <= 0 {
-		o.MemTableSize = 4 << 20
+		o.MemTableSize = 4 << 20 // 4 MB
 	}
 	if o.MemTableStopWritesThreshold <= 0 {
 		o.MemTableStopWritesThreshold = 2
@@ -1086,8 +1136,8 @@ func (o *Options) EnsureDefaults() *Options {
 	if o.Experimental.CPUWorkPermissionGranter == nil {
 		o.Experimental.CPUWorkPermissionGranter = defaultCPUWorkGranter{}
 	}
-	if o.Experimental.MultiLevelCompactionHueristic == nil {
+	if o.Experimental.MultiLevelCompactionHeuristic == nil {
-		o.Experimental.MultiLevelCompactionHueristic = NoMultiLevel{}
+		o.Experimental.MultiLevelCompactionHeuristic = WriteAmpHeuristic{}
 	}
 
 	o.initMaps()
@@ -1228,6 +1278,7 @@ func (o *Options) String() string {
 	fmt.Fprintf(&buf, "  max_writer_concurrency=%d\n", o.Experimental.MaxWriterConcurrency)
 	fmt.Fprintf(&buf, "  force_writer_parallelism=%t\n", o.Experimental.ForceWriterParallelism)
 	fmt.Fprintf(&buf, "  secondary_cache_size_bytes=%d\n", o.Experimental.SecondaryCacheSizeBytes)
+	fmt.Fprintf(&buf, "  create_on_shared=%d\n", o.Experimental.CreateOnShared)
 
 	// Private options.
 	//
@@ -1283,7 +1334,11 @@ func parseOptions(s string, fn func(section, key, value string) error) error {
 
 		pos := strings.Index(line, "=")
 		if pos < 0 {
-			return errors.Errorf("pebble: invalid key=value syntax: %s", errors.Safe(line))
+			const maxLen = 50
+			if len(line) > maxLen {
+				line = line[:maxLen-3] + "..."
+			}
+			return base.CorruptionErrorf("invalid key=value syntax: %q", errors.Safe(line))
 		}
 
 		key := strings.TrimSpace(line[:pos])
@@ -1499,6 +1554,10 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 				o.Experimental.ForceWriterParallelism, err = strconv.ParseBool(value)
 			case "secondary_cache_size_bytes":
 				o.Experimental.SecondaryCacheSizeBytes, err = strconv.ParseInt(value, 10, 64)
+			case "create_on_shared":
+				var createOnSharedInt int64
+				createOnSharedInt, err = strconv.ParseInt(value, 10, 64)
+				o.Experimental.CreateOnShared = remote.CreateOnSharedStrategy(createOnSharedInt)
 			default:
 				if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key, value) {
 					return nil

vendor/github.com/cockroachdb/pebble/range_keys.go

@@ -17,7 +17,7 @@ import (
 func (i *Iterator) constructRangeKeyIter() {
 	i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init(
 		&i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound,
-		&i.hasPrefix, &i.prefixOrFullSeekKey, true /* onlySets */, &i.rangeKey.rangeKeyBuffers.internal)
+		&i.hasPrefix, &i.prefixOrFullSeekKey, false /* internalKeys */, &i.rangeKey.rangeKeyBuffers.internal)
 
 	// If there's an indexed batch with range keys, include it.
 	if i.batch != nil {

vendor/github.com/cockroachdb/pebble/scan_internal.go

@@ -14,12 +14,13 @@ import (
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/remote"
 )
 
 const (
 	// In skip-shared iteration mode, keys in levels sharedLevelsStart and greater
 	// (i.e. lower in the LSM) are skipped.
-	sharedLevelsStart = 5
+	sharedLevelsStart = remote.SharedLevelsStart
 )
 
 // ErrInvalidSkipSharedIteration is returned by ScanInternal if it was called
@@ -661,7 +662,17 @@ func scanInternalImpl(
 		case InternalKeyKindRangeKeyDelete, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet:
 			if opts.visitRangeKey != nil {
 				span := iter.unsafeSpan()
-				if err := opts.visitRangeKey(span.Start, span.End, span.Keys); err != nil {
+				// NB: The caller isn't interested in the sequence numbers of these
+				// range keys. Rather, the caller wants them to be in trailer order
+				// _after_ zeroing of sequence numbers. Copy span.Keys, sort it, and then
+				// call visitRangeKey.
+				keysCopy := make([]keyspan.Key, len(span.Keys))
+				for i := range span.Keys {
+					keysCopy[i] = span.Keys[i]
+					keysCopy[i].Trailer = base.MakeTrailer(0, span.Keys[i].Kind())
+				}
+				keyspan.SortKeysByTrailer(&keysCopy)
+				if err := opts.visitRangeKey(span.Start, span.End, keysCopy); err != nil {
 					return err
 				}
 			}
@@ -763,7 +774,7 @@ func (i *scanInternalIterator) constructPointIter(memtables flushableList, buf *
 		rli := &rangeDelLevels[levelsIndex]
 
 		li.init(
-			context.Background(), i.opts.IterOptions, i.comparer.Compare, i.comparer.Split, i.newIters, files, level,
+			context.Background(), i.opts.IterOptions, i.comparer, i.newIters, files, level,
 			internalIterOpts{})
 		li.initBoundaryContext(&mlevels[mlevelsIndex].levelIterBoundaryContext)
 		mlevels[mlevelsIndex].iter = li
@@ -832,7 +843,7 @@ func (i *scanInternalIterator) constructRangeKeyIter() {
 	// RangeKeyUnsets and RangeKeyDels.
 	i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init(
 		i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound,
-		nil /* hasPrefix */, nil /* prefix */, false, /* onlySets */
+		nil /* hasPrefix */, nil /* prefix */, true, /* internalKeys */
 		&i.rangeKey.rangeKeyBuffers.internal)
 
 	// Next are the flushables: memtables and large batches.

vendor/github.com/cockroachdb/pebble/snapshot.go

@@ -205,7 +205,7 @@ func (l *snapshotList) remove(s *Snapshot) {
 }
 
 // EventuallyFileOnlySnapshot (aka EFOS) provides a read-only point-in-time view
-// of the database state, similar to Snapshot. A EventuallyFileOnlySnapshot
+// of the database state, similar to Snapshot. An EventuallyFileOnlySnapshot
 // induces less write amplification than Snapshot, at the cost of increased space
 // amplification. While a Snapshot may increase write amplification across all
 // flushes and compactions for the duration of its lifetime, an
@@ -225,28 +225,21 @@ func (l *snapshotList) remove(s *Snapshot) {
 // with zero write-amp and zero pinning of memtables in memory.
 //
 // EventuallyFileOnlySnapshots interact with the IngestAndExcise operation in
-// subtle ways. Unlike Snapshots, EFOS guarantees that their read-only
+// subtle ways. No new iterators can be created once
-// point-in-time view is unaltered by the excision. However, if a concurrent
+// EventuallyFileOnlySnapshot.excised is set to true.
-// excise were to happen on one of the protectedRanges, WaitForFileOnlySnapshot()
-// would return ErrSnapshotExcised and the EFOS would maintain a reference to the
-// underlying readState (and by extension, zombie memtables) for its lifetime.
-// This could lead to increased memory utilization, which is why callers should
-// call WaitForFileOnlySnapshot() if they expect an EFOS to be long-lived.
 type EventuallyFileOnlySnapshot struct {
 	mu struct {
 		// NB: If both this mutex and db.mu are being grabbed, db.mu should be
 		// grabbed _before_ grabbing this one.
 		sync.Mutex
 
-		// Either the {snap,readState} fields are set below, or the version is set at
+		// Either the snap field is set below, or the version is set at any given
-		// any given point of time. If a snapshot is referenced, this is not a
+		// point of time. If a snapshot is referenced, this is not a file-only
-		// file-only snapshot yet, and if a version is set (and ref'd) this is a
+		// snapshot yet, and if a version is set (and ref'd) this is a file-only
-		// file-only snapshot.
+		// snapshot.
 
-		// The wrapped regular snapshot, if not a file-only snapshot yet. The
+		// The wrapped regular snapshot, if not a file-only snapshot yet.
-		// readState has already been ref()d once if it's set.
+		snap *Snapshot
-		snap      *Snapshot
-		readState *readState
 		// The wrapped version reference, if a file-only snapshot.
 		vers *version
 	}
@@ -296,7 +289,6 @@ func (d *DB) makeEventuallyFileOnlySnapshot(
 		}
 		s.efos = es
 		es.mu.snap = s
-		es.mu.readState = d.loadReadState()
 		d.mu.snapshots.pushBack(s)
 	}
 	return es
@@ -328,33 +320,11 @@ func (es *EventuallyFileOnlySnapshot) transitionToFileOnlySnapshot(vers *version
 	es.mu.vers = vers
 	// NB: The callers should have already done a check of es.excised.
 	oldSnap := es.mu.snap
-	oldReadState := es.mu.readState
 	es.mu.snap = nil
-	es.mu.readState = nil
 	es.mu.Unlock()
-	// It's okay to close a snapshot even if iterators are already open on it.
-	oldReadState.unrefLocked()
 	return oldSnap.closeLocked()
 }
 
-// releaseReadState is called to release reference to a readState when
-// es.excised == true. This is to free up memory as quickly as possible; all
-// other snapshot resources are kept around until Close() is called. Safe for
-// idempotent calls.
-//
-// d.mu must be held when calling this method.
-func (es *EventuallyFileOnlySnapshot) releaseReadState() {
-	if !es.excised.Load() {
-		panic("pebble: releasing read state of eventually-file-only-snapshot but was not excised")
-	}
-	es.mu.Lock()
-	defer es.mu.Unlock()
-	if es.mu.readState != nil {
-		es.mu.readState.unrefLocked()
-		es.db.maybeScheduleObsoleteTableDeletionLocked()
-	}
-}
-
 // hasTransitioned returns true if this EFOS has transitioned to a file-only
 // snapshot.
 func (es *EventuallyFileOnlySnapshot) hasTransitioned() bool {
@@ -448,10 +418,6 @@ func (es *EventuallyFileOnlySnapshot) Close() error {
 			return err
 		}
 	}
-	if es.mu.readState != nil {
-		es.mu.readState.unrefLocked()
-		es.db.maybeScheduleObsoleteTableDeletionLocked()
-	}
 	if es.mu.vers != nil {
 		es.mu.vers.UnrefLocked()
 	}
@@ -511,8 +477,16 @@ func (es *EventuallyFileOnlySnapshot) NewIterWithContext(
 	if es.excised.Load() {
 		return nil, ErrSnapshotExcised
 	}
-	sOpts := snapshotIterOpts{seqNum: es.seqNum, readState: es.mu.readState}
+	sOpts := snapshotIterOpts{seqNum: es.seqNum}
-	return es.db.newIter(ctx, nil /* batch */, sOpts, o), nil
+	iter := es.db.newIter(ctx, nil /* batch */, sOpts, o)
+
+	// If excised is true, then keys relevant to the snapshot might not be
+	// present in the readState being used by the iterator. Error out.
+	if es.excised.Load() {
+		iter.Close()
+		return nil, ErrSnapshotExcised
+	}
+	return iter, nil
 }
 
 // ScanInternal scans all internal keys within the specified bounds, truncating
@@ -544,8 +518,7 @@ func (es *EventuallyFileOnlySnapshot) ScanInternal(
 		}
 	} else {
 		sOpts = snapshotIterOpts{
-			seqNum:    es.seqNum,
+			seqNum: es.seqNum,
-			readState: es.mu.readState,
 		}
 	}
 	es.mu.Unlock()
@@ -564,5 +537,11 @@ func (es *EventuallyFileOnlySnapshot) ScanInternal(
 	iter := es.db.newInternalIter(sOpts, opts)
 	defer iter.close()
 
+	// If excised is true, then keys relevant to the snapshot might not be
+	// present in the readState being used by the iterator. Error out.
+	if es.excised.Load() {
+		return ErrSnapshotExcised
+	}
+
 	return scanInternalImpl(ctx, lower, upper, iter, opts)
 }

vendor/github.com/cockroachdb/pebble/sstable/block_property.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"math"
 	"sync"
+	"unsafe"
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/rangekey"
@@ -643,15 +644,21 @@ func (f *BlockPropertiesFilterer) intersectsUserPropsAndFinishInit(
 			// considered intersecting.
 			continue
 		}
-		byteProps := []byte(props)
+		if len(props) < 1 {
-		if len(byteProps) < 1 {
 			return false, base.CorruptionErrorf(
 				"block properties for %s is corrupted", f.filters[i].Name())
 		}
-		shortID := shortID(byteProps[0])
+		shortID := shortID(props[0])
-		intersects, err := f.filters[i].Intersects(byteProps[1:])
+		{
-		if err != nil || !intersects {
+			// Use an unsafe conversion to avoid allocating. Intersects() is not
-			return false, err
+			// supposed to modify the given slice.
+			// Note that unsafe.StringData only works if the string is not empty
+			// (which we already checked).
+			byteProps := unsafe.Slice(unsafe.StringData(props), len(props))
+			intersects, err := f.filters[i].Intersects(byteProps[1:])
+			if err != nil || !intersects {
+				return false, err
+			}
 		}
 		// Intersects the sstable, so need to use this filter when
 		// deciding whether to read blocks.
@@ -684,12 +691,11 @@ func (f *BlockPropertiesFilterer) intersectsUserPropsAndFinishInit(
 		// be unused within this file.
 		return true, nil
 	}
-	byteProps := []byte(props)
+	if len(props) < 1 {
-	if len(byteProps) < 1 {
 		return false, base.CorruptionErrorf(
 			"block properties for %s is corrupted", f.boundLimitedFilter.Name())
 	}
-	f.boundLimitedShortID = int(byteProps[0])
+	f.boundLimitedShortID = int(props[0])
 
 	// We don't check for table-level intersection for the bound-limited filter.
 	// The bound-limited filter is treated as vacuously intersecting.

vendor/github.com/cockroachdb/pebble/sstable/properties.go

@@ -25,10 +25,20 @@ var propBoolFalse = []byte{'0'}
 
 var propOffsetTagMap = make(map[uintptr]string)
 
-func init() {
+func generateTagMaps(t reflect.Type, indexPrefix []int) {
-	t := reflect.TypeOf(Properties{})
 	for i := 0; i < t.NumField(); i++ {
 		f := t.Field(i)
+		if f.Type.Kind() == reflect.Struct {
+			if tag := f.Tag.Get("prop"); i == 0 && tag == "pebble.embbeded_common_properties" {
+				// CommonProperties struct embedded in Properties. Note that since
+				// CommonProperties is placed at the top of properties we can use
+				// the offsets of the fields within CommonProperties to determine
+				// the offsets of those fields within Properties.
+				generateTagMaps(f.Type, []int{i})
+				continue
+			}
+			panic("pebble: unknown struct type in Properties")
+		}
 		if tag := f.Tag.Get("prop"); tag != "" {
 			switch f.Type.Kind() {
 			case reflect.Bool:
@@ -38,16 +48,84 @@ func init() {
 			default:
 				panic(fmt.Sprintf("unsupported property field type: %s %s", f.Name, f.Type))
 			}
+			if len(indexPrefix) > 0 {
+				// Prepend the index prefix so that we can use FieldByIndex on the top-level struct.
+				f.Index = append(indexPrefix[:len(indexPrefix):len(indexPrefix)], f.Index...)
+			}
 			propTagMap[tag] = f
 			propOffsetTagMap[f.Offset] = tag
 		}
 	}
 }
 
+func init() {
+	generateTagMaps(reflect.TypeOf(Properties{}), nil)
+}
+
+// CommonProperties holds properties for either a virtual or a physical sstable. This
+// can be used by code which doesn't care to make the distinction between physical
+// and virtual sstables properties.
+//
+// For virtual sstables, fields are constructed through extrapolation upon virtual
+// reader construction. See MakeVirtualReader for implementation details.
+//
+// NB: The values of these properties can affect correctness. For example,
+// if NumRangeKeySets == 0, but the sstable actually contains range keys, then
+// the iterators will behave incorrectly.
+type CommonProperties struct {
+	// The number of entries in this table.
+	NumEntries uint64 `prop:"rocksdb.num.entries"`
+	// Total raw key size.
+	RawKeySize uint64 `prop:"rocksdb.raw.key.size"`
+	// Total raw value size.
+	RawValueSize uint64 `prop:"rocksdb.raw.value.size"`
+	// Total raw key size of point deletion tombstones. This value is comparable
+	// to RawKeySize.
+	RawPointTombstoneKeySize uint64 `prop:"pebble.raw.point-tombstone.key.size"`
+	// Sum of the raw value sizes carried by point deletion tombstones
+	// containing size estimates. See the DeleteSized key kind. This value is
+	// comparable to Raw{Key,Value}Size.
+	RawPointTombstoneValueSize uint64 `prop:"pebble.raw.point-tombstone.value.size"`
+	// The number of point deletion entries ("tombstones") in this table that
+	// carry a size hint indicating the size of the value the tombstone deletes.
+	NumSizedDeletions uint64 `prop:"pebble.num.deletions.sized"`
+	// The number of deletion entries in this table, including both point and
+	// range deletions.
+	NumDeletions uint64 `prop:"rocksdb.deleted.keys"`
+	// The number of range deletions in this table.
+	NumRangeDeletions uint64 `prop:"rocksdb.num.range-deletions"`
+	// The number of RANGEKEYDELs in this table.
+	NumRangeKeyDels uint64 `prop:"pebble.num.range-key-dels"`
+	// The number of RANGEKEYSETs in this table.
+	NumRangeKeySets uint64 `prop:"pebble.num.range-key-sets"`
+	// Total size of value blocks and value index block. Only serialized if > 0.
+	ValueBlocksSize uint64 `prop:"pebble.value-blocks.size"`
+}
+
+// String is only used for testing purposes.
+func (c *CommonProperties) String() string {
+	var buf bytes.Buffer
+	v := reflect.ValueOf(*c)
+	loaded := make(map[uintptr]struct{})
+	writeProperties(loaded, v, &buf)
+	return buf.String()
+}
+
+// NumPointDeletions is the number of point deletions in the sstable. For virtual
+// sstables, this is an estimate.
+func (c *CommonProperties) NumPointDeletions() uint64 {
+	return c.NumDeletions - c.NumRangeDeletions
+}
+
 // Properties holds the sstable property values. The properties are
 // automatically populated during sstable creation and load from the properties
 // meta block when an sstable is opened.
 type Properties struct {
+	// CommonProperties needs to be at the top of the Properties struct so that the
+	// offsets of the fields in CommonProperties match the offsets of the embedded
+	// fields of CommonProperties in Properties.
+	CommonProperties `prop:"pebble.embbeded_common_properties"`
+
 	// The name of the comparer used in this table.
 	ComparerName string `prop:"rocksdb.comparator"`
 	// The compression algorithm used to compress blocks.
@@ -81,22 +159,8 @@ type Properties struct {
 	MergerName string `prop:"rocksdb.merge.operator"`
 	// The number of blocks in this table.
 	NumDataBlocks uint64 `prop:"rocksdb.num.data.blocks"`
-	// The number of deletion entries in this table, including both point and
-	// range deletions.
-	NumDeletions uint64 `prop:"rocksdb.deleted.keys"`
-	// The number of point deletion entries ("tombstones") in this table that
-	// carry a size hint indicating the size of the value the tombstone deletes.
-	NumSizedDeletions uint64 `prop:"pebble.num.deletions.sized"`
-	// The number of entries in this table.
-	NumEntries uint64 `prop:"rocksdb.num.entries"`
 	// The number of merge operands in the table.
 	NumMergeOperands uint64 `prop:"rocksdb.merge.operands"`
-	// The number of range deletions in this table.
-	NumRangeDeletions uint64 `prop:"rocksdb.num.range-deletions"`
-	// The number of RANGEKEYDELs in this table.
-	NumRangeKeyDels uint64 `prop:"pebble.num.range-key-dels"`
-	// The number of RANGEKEYSETs in this table.
-	NumRangeKeySets uint64 `prop:"pebble.num.range-key-sets"`
 	// The number of RANGEKEYUNSETs in this table.
 	NumRangeKeyUnsets uint64 `prop:"pebble.num.range-key-unsets"`
 	// The number of value blocks in this table. Only serialized if > 0.
@@ -111,21 +175,10 @@ type Properties struct {
 	// A comma separated list of names of the property collectors used in this
 	// table.
 	PropertyCollectorNames string `prop:"rocksdb.property.collectors"`
-	// Total raw key size.
-	RawKeySize uint64 `prop:"rocksdb.raw.key.size"`
-	// Total raw key size of point deletion tombstones. This value is comparable
-	// to RawKeySize.
-	RawPointTombstoneKeySize uint64 `prop:"pebble.raw.point-tombstone.key.size"`
-	// Sum of the raw value sizes carried by point deletion tombstones
-	// containing size estimates. See the DeleteSized key kind. This value is
-	// comparable to Raw{Key,Value}Size.
-	RawPointTombstoneValueSize uint64 `prop:"pebble.raw.point-tombstone.value.size"`
 	// Total raw rangekey key size.
 	RawRangeKeyKeySize uint64 `prop:"pebble.raw.range-key.key.size"`
 	// Total raw rangekey value size.
 	RawRangeKeyValueSize uint64 `prop:"pebble.raw.range-key.value.size"`
-	// Total raw value size.
-	RawValueSize uint64 `prop:"rocksdb.raw.value.size"`
 	// The total number of keys in this table that were pinned by open snapshots.
 	SnapshotPinnedKeys uint64 `prop:"pebble.num.snapshot-pinned-keys"`
 	// The cumulative bytes of keys in this table that were pinned by
@@ -138,8 +191,6 @@ type Properties struct {
 	TopLevelIndexSize uint64 `prop:"rocksdb.top-level.index.size"`
 	// User collected properties.
 	UserProperties map[string]string
-	// Total size of value blocks and value index block. Only serialized if > 0.
-	ValueBlocksSize uint64 `prop:"pebble.value-blocks.size"`
 	// If filtering is enabled, was the filter created on the whole key.
 	WholeKeyFiltering bool `prop:"rocksdb.block.based.table.whole.key.filtering"`
 
@@ -160,12 +211,15 @@ func (p *Properties) NumRangeKeys() uint64 {
 	return p.NumRangeKeyDels + p.NumRangeKeySets + p.NumRangeKeyUnsets
 }
 
-func (p *Properties) String() string {
+func writeProperties(loaded map[uintptr]struct{}, v reflect.Value, buf *bytes.Buffer) {
-	var buf bytes.Buffer
-	v := reflect.ValueOf(*p)
 	vt := v.Type()
 	for i := 0; i < v.NumField(); i++ {
 		ft := vt.Field(i)
+		if ft.Type.Kind() == reflect.Struct {
+			// Embedded struct within the properties.
+			writeProperties(loaded, v.Field(i), buf)
+			continue
+		}
 		tag := ft.Tag.Get("prop")
 		if tag == "" {
 			continue
@@ -175,25 +229,33 @@ func (p *Properties) String() string {
 		// TODO(peter): Use f.IsZero() when we can rely on go1.13.
 		if zero := reflect.Zero(f.Type()); zero.Interface() == f.Interface() {
 			// Skip printing of zero values which were not loaded from disk.
-			if _, ok := p.Loaded[ft.Offset]; !ok {
+			if _, ok := loaded[ft.Offset]; !ok {
 				continue
 			}
 		}
 
-		fmt.Fprintf(&buf, "%s: ", tag)
+		fmt.Fprintf(buf, "%s: ", tag)
 		switch ft.Type.Kind() {
 		case reflect.Bool:
-			fmt.Fprintf(&buf, "%t\n", f.Bool())
+			fmt.Fprintf(buf, "%t\n", f.Bool())
 		case reflect.Uint32:
-			fmt.Fprintf(&buf, "%d\n", f.Uint())
+			fmt.Fprintf(buf, "%d\n", f.Uint())
 		case reflect.Uint64:
-			fmt.Fprintf(&buf, "%d\n", f.Uint())
+			fmt.Fprintf(buf, "%d\n", f.Uint())
 		case reflect.String:
-			fmt.Fprintf(&buf, "%s\n", f.String())
+			fmt.Fprintf(buf, "%s\n", f.String())
 		default:
 			panic("not reached")
 		}
 	}
+}
+
+func (p *Properties) String() string {
+	var buf bytes.Buffer
+	v := reflect.ValueOf(*p)
+	writeProperties(p.Loaded, v, &buf)
+
+	// Write the UserProperties.
 	keys := make([]string, 0, len(p.UserProperties))
 	for key := range p.UserProperties {
 		keys = append(keys, key)
@@ -214,6 +276,7 @@ func (p *Properties) load(
 	}
 	p.Loaded = make(map[uintptr]struct{})
 	v := reflect.ValueOf(p).Elem()
+
 	for valid := i.First(); valid; valid = i.Next() {
 		if f, ok := propTagMap[string(i.Key().UserKey)]; ok {
 			p.Loaded[f.Offset] = struct{}{}

vendor/github.com/cockroachdb/pebble/sstable/reader.go

@@ -174,6 +174,28 @@ func init() {
 	private.SSTableRawTombstonesOpt = rawTombstonesOpt{}
 }
 
+// CommonReader abstracts functionality over a Reader or a VirtualReader. This
+// can be used by code which doesn't care to distinguish between a reader and a
+// virtual reader.
+type CommonReader interface {
+	NewRawRangeKeyIter() (keyspan.FragmentIterator, error)
+	NewRawRangeDelIter() (keyspan.FragmentIterator, error)
+	NewIterWithBlockPropertyFiltersAndContextEtc(
+		ctx context.Context, lower, upper []byte,
+		filterer *BlockPropertiesFilterer,
+		hideObsoletePoints, useFilterBlock bool,
+		stats *base.InternalIteratorStats,
+		rp ReaderProvider,
+	) (Iterator, error)
+	NewCompactionIter(
+		bytesIterated *uint64,
+		rp ReaderProvider,
+		bufferPool *BufferPool,
+	) (Iterator, error)
+	EstimateDiskUsage(start, end []byte) (uint64, error)
+	CommonProperties() *CommonProperties
+}
+
 // Reader is a table reader.
 type Reader struct {
 	readable          objstorage.Readable
@@ -337,7 +359,7 @@ func (r *Reader) newCompactionIter(
 		err := i.init(
 			context.Background(),
 			r, v, nil /* lower */, nil /* upper */, nil,
-			false /* useFilter */, false, /* hideObsoletePoints */
+			false /* useFilter */, v != nil && v.isForeign, /* hideObsoletePoints */
 			nil /* stats */, rp, bufferPool,
 		)
 		if err != nil {
@@ -352,7 +374,7 @@ func (r *Reader) newCompactionIter(
 	i := singleLevelIterPool.Get().(*singleLevelIterator)
 	err := i.init(
 		context.Background(), r, v, nil /* lower */, nil, /* upper */
-		nil, false /* useFilter */, false, /* hideObsoletePoints */
+		nil, false /* useFilter */, v != nil && v.isForeign, /* hideObsoletePoints */
 		nil /* stats */, rp, bufferPool,
 	)
 	if err != nil {
@@ -927,6 +949,11 @@ func (r *Reader) ValidateBlockChecksums() error {
 	return nil
 }
 
+// CommonProperties implemented the CommonReader interface.
+func (r *Reader) CommonProperties() *CommonProperties {
+	return &r.Properties.CommonProperties
+}
+
 // EstimateDiskUsage returns the total size of data blocks overlapping the range
 // `[start, end]`. Even if a data block partially overlaps, or we cannot
 // determine overlap due to abbreviated index keys, the full data block size is

vendor/github.com/cockroachdb/pebble/sstable/reader_iter_two_lvl.go

@@ -810,6 +810,11 @@ func (i *twoLevelIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyVa
 	if key, val := i.singleLevelIterator.NextPrefix(succKey); key != nil {
 		return key, val
 	}
+	// key == nil
+	if i.err != nil {
+		return nil, base.LazyValue{}
+	}
+
 	// Did not find prefix in the existing second-level index block. This is the
 	// slow-path where we seek the iterator.
 	var ikey *InternalKey

vendor/github.com/cockroachdb/pebble/sstable/reader_virtual.go

@@ -22,46 +22,58 @@ import (
 type VirtualReader struct {
 	vState     virtualState
 	reader     *Reader
-	Properties struct {
+	Properties CommonProperties
-		// RawKeySize, RawValueSize are set upon construction of a
-		// VirtualReader. The values of the fields is extrapolated. See
-		// MakeVirtualReader for implementation details.
-		RawKeySize   uint64
-		RawValueSize uint64
-	}
 }
 
 // Lightweight virtual sstable state which can be passed to sstable iterators.
 type virtualState struct {
-	lower   InternalKey
+	lower     InternalKey
-	upper   InternalKey
+	upper     InternalKey
-	fileNum base.FileNum
+	fileNum   base.FileNum
-	Compare Compare
+	Compare   Compare
+	isForeign bool
+}
+
+func ceilDiv(a, b uint64) uint64 {
+	return (a + b - 1) / b
 }
 
 // MakeVirtualReader is used to contruct a reader which can read from virtual
 // sstables.
-func MakeVirtualReader(reader *Reader, meta manifest.VirtualFileMeta) VirtualReader {
+func MakeVirtualReader(
+	reader *Reader, meta manifest.VirtualFileMeta, isForeign bool,
+) VirtualReader {
 	if reader.fileNum != meta.FileBacking.DiskFileNum {
 		panic("pebble: invalid call to MakeVirtualReader")
 	}
 
 	vState := virtualState{
-		lower:   meta.Smallest,
+		lower:     meta.Smallest,
-		upper:   meta.Largest,
+		upper:     meta.Largest,
-		fileNum: meta.FileNum,
+		fileNum:   meta.FileNum,
-		Compare: reader.Compare,
+		Compare:   reader.Compare,
+		isForeign: isForeign,
 	}
 	v := VirtualReader{
 		vState: vState,
 		reader: reader,
 	}
 
-	v.Properties.RawKeySize =
+	v.Properties.RawKeySize = ceilDiv(reader.Properties.RawKeySize*meta.Size, meta.FileBacking.Size)
-		(reader.Properties.RawKeySize * meta.Size) / meta.FileBacking.Size
+	v.Properties.RawValueSize = ceilDiv(reader.Properties.RawValueSize*meta.Size, meta.FileBacking.Size)
-	v.Properties.RawValueSize =
+	v.Properties.NumEntries = ceilDiv(reader.Properties.NumEntries*meta.Size, meta.FileBacking.Size)
-		(reader.Properties.RawValueSize * meta.Size) / meta.FileBacking.Size
+	v.Properties.NumDeletions = ceilDiv(reader.Properties.NumDeletions*meta.Size, meta.FileBacking.Size)
+	v.Properties.NumRangeDeletions = ceilDiv(reader.Properties.NumRangeDeletions*meta.Size, meta.FileBacking.Size)
+	v.Properties.NumRangeKeyDels = ceilDiv(reader.Properties.NumRangeKeyDels*meta.Size, meta.FileBacking.Size)
 
+	// Note that we rely on NumRangeKeySets for correctness. If the sstable may
+	// contain range keys, then NumRangeKeySets must be > 0. ceilDiv works because
+	// meta.Size will not be 0 for virtual sstables.
+	v.Properties.NumRangeKeySets = ceilDiv(reader.Properties.NumRangeKeySets*meta.Size, meta.FileBacking.Size)
+	v.Properties.ValueBlocksSize = ceilDiv(reader.Properties.ValueBlocksSize*meta.Size, meta.FileBacking.Size)
+	v.Properties.NumSizedDeletions = ceilDiv(reader.Properties.NumSizedDeletions*meta.Size, meta.FileBacking.Size)
+	v.Properties.RawPointTombstoneKeySize = ceilDiv(reader.Properties.RawPointTombstoneKeySize*meta.Size, meta.FileBacking.Size)
+	v.Properties.RawPointTombstoneValueSize = ceilDiv(reader.Properties.RawPointTombstoneValueSize*meta.Size, meta.FileBacking.Size)
 	return v
 }
 
@@ -89,6 +101,12 @@ func (v *VirtualReader) NewIterWithBlockPropertyFiltersAndContextEtc(
 	)
 }
 
+// ValidateBlockChecksumsOnBacking will call ValidateBlockChecksumsOnBacking on the underlying reader.
+// Note that block checksum validation is NOT restricted to virtual sstable bounds.
+func (v *VirtualReader) ValidateBlockChecksumsOnBacking() error {
+	return v.reader.ValidateBlockChecksums()
+}
+
 // NewRawRangeDelIter wraps Reader.NewRawRangeDelIter.
 func (v *VirtualReader) NewRawRangeDelIter() (keyspan.FragmentIterator, error) {
 	iter, err := v.reader.NewRawRangeDelIter()
@@ -181,3 +199,8 @@ func (v *VirtualReader) EstimateDiskUsage(start, end []byte) (uint64, error) {
 	_, f, l := v.vState.constrainBounds(start, end, true /* endInclusive */)
 	return v.reader.EstimateDiskUsage(f, l)
 }
+
+// CommonProperties implements the CommonReader interface.
+func (v *VirtualReader) CommonProperties() *CommonProperties {
+	return &v.Properties
+}

vendor/github.com/cockroachdb/pebble/table_cache.go

@@ -194,6 +194,40 @@ func (c *tableCacheContainer) estimateSize(
 	return size, nil
 }
 
+// createCommonReader creates a Reader for this file. isForeign, if true for
+// virtual sstables, is passed into the vSSTable reader so its iterators can
+// collapse obsolete points accordingly.
+func createCommonReader(
+	v *tableCacheValue, file *fileMetadata, isForeign bool,
+) sstable.CommonReader {
+	// TODO(bananabrick): We suffer an allocation if file is a virtual sstable.
+	var cr sstable.CommonReader = v.reader
+	if file.Virtual {
+		virtualReader := sstable.MakeVirtualReader(
+			v.reader, file.VirtualMeta(), isForeign,
+		)
+		cr = &virtualReader
+	}
+	return cr
+}
+
+func (c *tableCacheContainer) withCommonReader(
+	meta *fileMetadata, fn func(sstable.CommonReader) error,
+) error {
+	s := c.tableCache.getShard(meta.FileBacking.DiskFileNum)
+	v := s.findNode(meta, &c.dbOpts)
+	defer s.unrefValue(v)
+	if v.err != nil {
+		return v.err
+	}
+	provider := c.dbOpts.objProvider
+	objMeta, err := provider.Lookup(fileTypeTable, meta.FileBacking.DiskFileNum)
+	if err != nil {
+		return err
+	}
+	return fn(createCommonReader(v, meta, provider.IsSharedForeign(objMeta)))
+}
+
 func (c *tableCacheContainer) withReader(meta physicalMeta, fn func(*sstable.Reader) error) error {
 	s := c.tableCache.getShard(meta.FileBacking.DiskFileNum)
 	v := s.findNode(meta.FileMetadata, &c.dbOpts)
@@ -214,7 +248,12 @@ func (c *tableCacheContainer) withVirtualReader(
 	if v.err != nil {
 		return v.err
 	}
-	return fn(sstable.MakeVirtualReader(v.reader, meta))
+	provider := c.dbOpts.objProvider
+	objMeta, err := provider.Lookup(fileTypeTable, meta.FileBacking.DiskFileNum)
+	if err != nil {
+		return err
+	}
+	return fn(sstable.MakeVirtualReader(v.reader, meta, provider.IsSharedForeign(objMeta)))
 }
 
 func (c *tableCacheContainer) iterCount() int64 {
@@ -432,25 +471,6 @@ func (c *tableCacheShard) newIters(
 		return nil, nil, err
 	}
 
-	type iterCreator interface {
-		NewRawRangeDelIter() (keyspan.FragmentIterator, error)
-		NewIterWithBlockPropertyFiltersAndContextEtc(ctx context.Context, lower, upper []byte, filterer *sstable.BlockPropertiesFilterer, hideObsoletePoints, useFilterBlock bool, stats *base.InternalIteratorStats, rp sstable.ReaderProvider) (sstable.Iterator, error)
-		NewCompactionIter(
-			bytesIterated *uint64,
-			rp sstable.ReaderProvider,
-			bufferPool *sstable.BufferPool,
-		) (sstable.Iterator, error)
-	}
-
-	// TODO(bananabrick): We suffer an allocation if file is a virtual sstable.
-	var ic iterCreator = v.reader
-	if file.Virtual {
-		virtualReader := sstable.MakeVirtualReader(
-			v.reader, file.VirtualMeta(),
-		)
-		ic = &virtualReader
-	}
-
 	provider := dbOpts.objProvider
 	// Check if this file is a foreign file.
 	objMeta, err := provider.Lookup(fileTypeTable, file.FileBacking.DiskFileNum)
@@ -458,9 +478,12 @@ func (c *tableCacheShard) newIters(
 		return nil, nil, err
 	}
 
+	// Note: This suffers an allocation for virtual sstables.
+	cr := createCommonReader(v, file, provider.IsSharedForeign(objMeta))
+
 	// NB: range-del iterator does not maintain a reference to the table, nor
 	// does it need to read from it after creation.
-	rangeDelIter, err := ic.NewRawRangeDelIter()
+	rangeDelIter, err := cr.NewRawRangeDelIter()
 	if err != nil {
 		c.unrefValue(v)
 		return nil, nil, err
@@ -504,9 +527,9 @@ func (c *tableCacheShard) newIters(
 		hideObsoletePoints = true
 	}
 	if internalOpts.bytesIterated != nil {
-		iter, err = ic.NewCompactionIter(internalOpts.bytesIterated, rp, internalOpts.bufferPool)
+		iter, err = cr.NewCompactionIter(internalOpts.bytesIterated, rp, internalOpts.bufferPool)
 	} else {
-		iter, err = ic.NewIterWithBlockPropertyFiltersAndContextEtc(
+		iter, err = cr.NewIterWithBlockPropertyFiltersAndContextEtc(
 			ctx, opts.GetLowerBound(), opts.GetUpperBound(), filterer, hideObsoletePoints, useFilter,
 			internalOpts.stats, rp)
 	}
@@ -567,10 +590,15 @@ func (c *tableCacheShard) newRangeKeyIter(
 
 	var iter keyspan.FragmentIterator
 	if file.Virtual {
-		virtualReader := sstable.MakeVirtualReader(
+		provider := dbOpts.objProvider
-			v.reader, file.VirtualMeta(),
+		var objMeta objstorage.ObjectMetadata
-		)
+		objMeta, err = provider.Lookup(fileTypeTable, file.FileBacking.DiskFileNum)
-		iter, err = virtualReader.NewRawRangeKeyIter()
+		if err == nil {
+			virtualReader := sstable.MakeVirtualReader(
+				v.reader, file.VirtualMeta(), provider.IsSharedForeign(objMeta),
+			)
+			iter, err = virtualReader.NewRawRangeKeyIter()
+		}
 	} else {
 		iter, err = v.reader.NewRawRangeKeyIter()
 	}
@@ -714,23 +742,30 @@ func (c *tableCacheShard) unrefValue(v *tableCacheValue) {
 // findNode returns the node for the table with the given file number, creating
 // that node if it didn't already exist. The caller is responsible for
 // decrementing the returned node's refCount.
-func (c *tableCacheShard) findNode(
+func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) *tableCacheValue {
-	meta *fileMetadata, dbOpts *tableCacheOpts,
+	v := c.findNodeInternal(meta, dbOpts)
-) (v *tableCacheValue) {
+
 	// Loading a file before its global sequence number is known (eg,
 	// during ingest before entering the commit pipeline) can pollute
 	// the cache with incorrect state. In invariant builds, verify
 	// that the global sequence number of the returned reader matches.
 	if invariants.Enabled {
-		defer func() {
+		if v.reader != nil && meta.LargestSeqNum == meta.SmallestSeqNum &&
-			if v.reader != nil && meta.LargestSeqNum == meta.SmallestSeqNum &&
+			v.reader.Properties.GlobalSeqNum != meta.SmallestSeqNum {
-				v.reader.Properties.GlobalSeqNum != meta.SmallestSeqNum {
+			panic(errors.AssertionFailedf("file %s loaded from table cache with the wrong global sequence number %d",
-				panic(errors.AssertionFailedf("file %s loaded from table cache with the wrong global sequence number %d",
+				meta, v.reader.Properties.GlobalSeqNum))
-					meta, v.reader.Properties.GlobalSeqNum))
+		}
-			}
-		}()
 	}
+	return v
+}
 
+func (c *tableCacheShard) findNodeInternal(
+	meta *fileMetadata, dbOpts *tableCacheOpts,
+) *tableCacheValue {
+	if refs := meta.Refs(); refs <= 0 {
+		panic(errors.AssertionFailedf("attempting to load file %s with refs=%d from table cache",
+			meta, refs))
+	}
 	// Fast-path for a hit in the cache.
 	c.mu.RLock()
 	key := tableCacheKey{dbOpts.cacheID, meta.FileBacking.DiskFileNum}
@@ -738,7 +773,7 @@ func (c *tableCacheShard) findNode(
 		// Fast-path hit.
 		//
 		// The caller is responsible for decrementing the refCount.
-		v = n.value
+		v := n.value
 		v.refCount.Add(1)
 		c.mu.RUnlock()
 		n.referenced.Store(true)
@@ -765,7 +800,7 @@ func (c *tableCacheShard) findNode(
 		// Slow-path hit of a hot or cold node.
 		//
 		// The caller is responsible for decrementing the refCount.
-		v = n.value
+		v := n.value
 		v.refCount.Add(1)
 		n.referenced.Store(true)
 		c.hits.Add(1)
@@ -789,7 +824,7 @@ func (c *tableCacheShard) findNode(
 
 	c.misses.Add(1)
 
-	v = &tableCacheValue{
+	v := &tableCacheValue{
 		loaded: make(chan struct{}),
 	}
 	v.refCount.Store(2)

vendor/github.com/cockroachdb/pebble/table_stats.go

@@ -186,14 +186,9 @@ func (d *DB) loadNewFileStats(
 			continue
 		}
 
-		if nf.Meta.Virtual {
-			// cannot load virtual table stats
-			continue
-		}
-
 		stats, newHints, err := d.loadTableStats(
 			rs.current, nf.Level,
-			nf.Meta.PhysicalMeta(),
+			nf.Meta,
 		)
 		if err != nil {
 			d.opts.EventListener.BackgroundError(err)
@@ -222,12 +217,6 @@ func (d *DB) scanReadStateTableStats(
 	for l, levelMetadata := range rs.current.Levels {
 		iter := levelMetadata.Iter()
 		for f := iter.First(); f != nil; f = iter.Next() {
-			if f.Virtual {
-				// TODO(bananabrick): Support stats collection for virtual
-				// sstables.
-				continue
-			}
-
 			// NB: We're not holding d.mu which protects f.Stats, but only the
 			// active stats collection job updates f.Stats for active files,
 			// and we ensure only one goroutine runs it at a time through
@@ -236,11 +225,6 @@ func (d *DB) scanReadStateTableStats(
 			if f.StatsValid() {
 				continue
 			}
-			// TODO(bilal): Remove this guard when table stats collection is
-			// implemented for virtual sstables.
-			if f.Virtual {
-				continue
-			}
 
 			// Limit how much work we do per read state. The older the read
 			// state is, the higher the likelihood files are no longer being
@@ -253,7 +237,7 @@ func (d *DB) scanReadStateTableStats(
 			}
 
 			stats, newHints, err := d.loadTableStats(
-				rs.current, l, f.PhysicalMeta(),
+				rs.current, l, f,
 			)
 			if err != nil {
 				// Set `moreRemain` so we'll try again.
@@ -271,25 +255,22 @@ func (d *DB) scanReadStateTableStats(
 	return fill, hints, moreRemain
 }
 
-// loadTableStats currently only supports stats collection for physical
-// sstables.
-//
-// TODO(bananabrick): Support stats collection for virtual sstables.
 func (d *DB) loadTableStats(
-	v *version, level int, meta physicalMeta,
+	v *version, level int, meta *fileMetadata,
 ) (manifest.TableStats, []deleteCompactionHint, error) {
 	var stats manifest.TableStats
 	var compactionHints []deleteCompactionHint
-	err := d.tableCache.withReader(
+	err := d.tableCache.withCommonReader(
-		meta, func(r *sstable.Reader) (err error) {
+		meta, func(r sstable.CommonReader) (err error) {
-			stats.NumEntries = r.Properties.NumEntries
+			props := r.CommonProperties()
-			stats.NumDeletions = r.Properties.NumDeletions
+			stats.NumEntries = props.NumEntries
-			if r.Properties.NumPointDeletions() > 0 {
+			stats.NumDeletions = props.NumDeletions
-				if err = d.loadTablePointKeyStats(r, v, level, meta, &stats); err != nil {
+			if props.NumPointDeletions() > 0 {
+				if err = d.loadTablePointKeyStats(props, v, level, meta, &stats); err != nil {
 					return
 				}
 			}
-			if r.Properties.NumRangeDeletions > 0 || r.Properties.NumRangeKeyDels > 0 {
+			if props.NumRangeDeletions > 0 || props.NumRangeKeyDels > 0 {
 				if compactionHints, err = d.loadTableRangeDelStats(
 					r, v, level, meta, &stats,
 				); err != nil {
@@ -299,8 +280,8 @@ func (d *DB) loadTableStats(
 			// TODO(travers): Once we have real-world data, consider collecting
 			// additional stats that may provide improved heuristics for compaction
 			// picking.
-			stats.NumRangeKeySets = r.Properties.NumRangeKeySets
+			stats.NumRangeKeySets = props.NumRangeKeySets
-			stats.ValueBlocksSize = r.Properties.ValueBlocksSize
+			stats.ValueBlocksSize = props.ValueBlocksSize
 			return
 		})
 	if err != nil {
@@ -312,7 +293,11 @@ func (d *DB) loadTableStats(
 // loadTablePointKeyStats calculates the point key statistics for the given
 // table. The provided manifest.TableStats are updated.
 func (d *DB) loadTablePointKeyStats(
-	r *sstable.Reader, v *version, level int, meta physicalMeta, stats *manifest.TableStats,
+	props *sstable.CommonProperties,
+	v *version,
+	level int,
+	meta *fileMetadata,
+	stats *manifest.TableStats,
 ) error {
 	// TODO(jackson): If the file has a wide keyspace, the average
 	// value size beneath the entire file might not be representative
@@ -320,21 +305,21 @@ func (d *DB) loadTablePointKeyStats(
 	// We could write the ranges of 'clusters' of point tombstones to
 	// a sstable property and call averageValueSizeBeneath for each of
 	// these narrower ranges to improve the estimate.
-	avgValLogicalSize, compressionRatio, err := d.estimateSizesBeneath(v, level, meta)
+	avgValLogicalSize, compressionRatio, err := d.estimateSizesBeneath(v, level, meta, props)
 	if err != nil {
 		return err
 	}
 	stats.PointDeletionsBytesEstimate =
-		pointDeletionsBytesEstimate(meta.Size, &r.Properties, avgValLogicalSize, compressionRatio)
+		pointDeletionsBytesEstimate(meta.Size, props, avgValLogicalSize, compressionRatio)
 	return nil
 }
 
 // loadTableRangeDelStats calculates the range deletion and range key deletion
 // statistics for the given table.
 func (d *DB) loadTableRangeDelStats(
-	r *sstable.Reader, v *version, level int, meta physicalMeta, stats *manifest.TableStats,
+	r sstable.CommonReader, v *version, level int, meta *fileMetadata, stats *manifest.TableStats,
 ) ([]deleteCompactionHint, error) {
-	iter, err := newCombinedDeletionKeyspanIter(d.opts.Comparer, r, meta.FileMetadata)
+	iter, err := newCombinedDeletionKeyspanIter(d.opts.Comparer, r, meta)
 	if err != nil {
 		return nil, err
 	}
@@ -423,7 +408,7 @@ func (d *DB) loadTableRangeDelStats(
 			hintType:                hintType,
 			start:                   make([]byte, len(start)),
 			end:                     make([]byte, len(end)),
-			tombstoneFile:           meta.FileMetadata,
+			tombstoneFile:           meta,
 			tombstoneLevel:          level,
 			tombstoneLargestSeqNum:  s.LargestSeqNum(),
 			tombstoneSmallestSeqNum: s.SmallestSeqNum(),
@@ -437,12 +422,21 @@ func (d *DB) loadTableRangeDelStats(
 }
 
 func (d *DB) estimateSizesBeneath(
-	v *version, level int, meta physicalMeta,
+	v *version, level int, meta *fileMetadata, fileProps *sstable.CommonProperties,
 ) (avgValueLogicalSize, compressionRatio float64, err error) {
 	// Find all files in lower levels that overlap with meta,
 	// summing their value sizes and entry counts.
-	file := meta.FileMetadata
+	file := meta
 	var fileSum, keySum, valSum, entryCount uint64
+	// Include the file itself. This is important because in some instances, the
+	// computed compression ratio is applied to the tombstones contained within
+	// `meta` itself. If there are no files beneath `meta` in the LSM, we would
+	// calculate a compression ratio of 0 which is not accurate for the file's
+	// own tombstones.
+	fileSum += file.Size
+	entryCount += fileProps.NumEntries
+	keySum += fileProps.RawKeySize
+	valSum += fileProps.RawValueSize
 
 	addPhysicalTableStats := func(r *sstable.Reader) (err error) {
 		fileSum += file.Size
@@ -459,15 +453,6 @@ func (d *DB) estimateSizesBeneath(
 		return nil
 	}
 
-	// Include the file itself. This is important because in some instances, the
-	// computed compression ratio is applied to the tombstones contained within
-	// `meta` itself. If there are no files beneath `meta` in the LSM, we would
-	// calculate a compression ratio of 0 which is not accurate for the file's
-	// own tombstones.
-	if err = d.tableCache.withReader(meta, addPhysicalTableStats); err != nil {
-		return 0, 0, err
-	}
-
 	for l := level + 1; l < numLevels; l++ {
 		overlaps := v.Overlaps(l, d.cmp, meta.Smallest.UserKey,
 			meta.Largest.UserKey, meta.Largest.IsExclusiveSentinel())
@@ -637,8 +622,9 @@ func maybeSetStatsFromProperties(meta physicalMeta, props *sstable.Properties) b
 		// doesn't require any additional IO and since the number of point
 		// deletions in the file is low, the error introduced by this crude
 		// estimate is expected to be small.
-		avgValSize, compressionRatio := estimatePhysicalSizes(meta.Size, props)
+		commonProps := &props.CommonProperties
-		pointEstimate = pointDeletionsBytesEstimate(meta.Size, props, avgValSize, compressionRatio)
+		avgValSize, compressionRatio := estimatePhysicalSizes(meta.Size, commonProps)
+		pointEstimate = pointDeletionsBytesEstimate(meta.Size, commonProps, avgValSize, compressionRatio)
 	}
 
 	meta.Stats.NumEntries = props.NumEntries
@@ -652,7 +638,7 @@ func maybeSetStatsFromProperties(meta physicalMeta, props *sstable.Properties) b
 }
 
 func pointDeletionsBytesEstimate(
-	fileSize uint64, props *sstable.Properties, avgValLogicalSize, compressionRatio float64,
+	fileSize uint64, props *sstable.CommonProperties, avgValLogicalSize, compressionRatio float64,
 ) (estimate uint64) {
 	if props.NumEntries == 0 {
 		return 0
@@ -675,7 +661,7 @@ func pointDeletionsBytesEstimate(
 	// tombstones' encoded values.
 	//
 	// For un-sized point tombstones (DELs), we estimate assuming that each
-	// point tombstone on average covers 1 key and using average vaue sizes.
+	// point tombstone on average covers 1 key and using average value sizes.
 	// This is almost certainly an overestimate, but that's probably okay
 	// because point tombstones can slow range iterations even when they don't
 	// cover a key.
@@ -739,7 +725,7 @@ func pointDeletionsBytesEstimate(
 }
 
 func estimatePhysicalSizes(
-	fileSize uint64, props *sstable.Properties,
+	fileSize uint64, props *sstable.CommonProperties,
 ) (avgValLogicalSize, compressionRatio float64) {
 	// RawKeySize and RawValueSize are uncompressed totals. Scale according to
 	// the data size to account for compression, index blocks and metadata
@@ -812,7 +798,7 @@ func estimatePhysicalSizes(
 // corresponding to the largest and smallest sequence numbers encountered across
 // the range deletes and range keys deletes that comprised the merged spans.
 func newCombinedDeletionKeyspanIter(
-	comparer *base.Comparer, r *sstable.Reader, m *fileMetadata,
+	comparer *base.Comparer, cr sstable.CommonReader, m *fileMetadata,
 ) (keyspan.FragmentIterator, error) {
 	// The range del iter and range key iter are each wrapped in their own
 	// defragmenting iter. For each iter, abutting spans can always be merged.
@@ -874,7 +860,7 @@ func newCombinedDeletionKeyspanIter(
 	})
 	mIter.Init(comparer.Compare, transform, new(keyspan.MergingBuffers))
 
-	iter, err := r.NewRawRangeDelIter()
+	iter, err := cr.NewRawRangeDelIter()
 	if err != nil {
 		return nil, err
 	}
@@ -891,7 +877,7 @@ func newCombinedDeletionKeyspanIter(
 		mIter.AddLevel(iter)
 	}
 
-	iter, err = r.NewRawRangeKeyIter()
+	iter, err = cr.NewRawRangeKeyIter()
 	if err != nil {
 		return nil, err
 	}
@@ -908,7 +894,7 @@ func newCombinedDeletionKeyspanIter(
 				out.Keys = append(out.Keys, k)
 			}
 			return len(out.Keys) > 0
-		})
+		}, comparer.Compare)
 		dIter := &keyspan.DefragmentingIter{}
 		dIter.Init(comparer, iter, equal, reducer, new(keyspan.DefragmentingBuffers))
 		iter = dIter

vendor/github.com/cockroachdb/pebble/version_set.go

@@ -86,17 +86,21 @@ type versionSet struct {
 	// still referenced by an inuse iterator.
 	zombieTables map[base.DiskFileNum]uint64 // filenum -> size
 
-	// fileBackingMap is a map for the FileBacking which is supporting virtual
+	// backingState is protected by the versionSet.logLock. It's populated
-	// sstables in the latest version. Once the file backing is backing no
-	// virtual sstables in the latest version, it is removed from this map and
-	// the corresponding state is added to the zombieTables map. Note that we
-	// don't keep track of file backing which supports a virtual sstable
-	// which is not in the latest version.
-	//
-	// fileBackingMap is protected by the versionSet.logLock. It's populated
 	// during Open in versionSet.load, but it's not used concurrently during
 	// load.
-	fileBackingMap map[base.DiskFileNum]*fileBacking
+	backingState struct {
+		// fileBackingMap is a map for the FileBacking which is supporting virtual
+		// sstables in the latest version. Once the file backing is backing no
+		// virtual sstables in the latest version, it is removed from this map and
+		// the corresponding state is added to the zombieTables map. Note that we
+		// don't keep track of file backing which supports a virtual sstable
+		// which is not in the latest version.
+		fileBackingMap map[base.DiskFileNum]*fileBacking
+		// fileBackingSize is the sum of the sizes of the fileBackings in the
+		// fileBackingMap.
+		fileBackingSize uint64
+	}
 
 	// minUnflushedLogNum is the smallest WAL log file number corresponding to
 	// mutations that have not been flushed to an sstable.
@@ -110,9 +114,10 @@ type versionSet struct {
 	manifestFileNum FileNum
 	manifestMarker  *atomicfs.Marker
 
-	manifestFile vfs.File
+	manifestFile          vfs.File
-	manifest     *record.Writer
+	manifest              *record.Writer
-	setCurrent   func(FileNum) error
+	setCurrent            func(FileNum) error
+	getFormatMajorVersion func() FormatMajorVersion
 
 	writing    bool
 	writerCond sync.Cond
@@ -125,6 +130,7 @@ func (vs *versionSet) init(
 	opts *Options,
 	marker *atomicfs.Marker,
 	setCurrent func(FileNum) error,
+	getFMV func() FormatMajorVersion,
 	mu *sync.Mutex,
 ) {
 	vs.dirname = dirname
@@ -138,10 +144,12 @@ func (vs *versionSet) init(
 	vs.versions.Init(mu)
 	vs.obsoleteFn = vs.addObsoleteLocked
 	vs.zombieTables = make(map[base.DiskFileNum]uint64)
-	vs.fileBackingMap = make(map[base.DiskFileNum]*fileBacking)
+	vs.backingState.fileBackingMap = make(map[base.DiskFileNum]*fileBacking)
+	vs.backingState.fileBackingSize = 0
 	vs.nextFileNum = 1
 	vs.manifestMarker = marker
 	vs.setCurrent = setCurrent
+	vs.getFormatMajorVersion = getFMV
 }
 
 // create creates a version set for a fresh DB.
@@ -151,9 +159,10 @@ func (vs *versionSet) create(
 	opts *Options,
 	marker *atomicfs.Marker,
 	setCurrent func(FileNum) error,
+	getFormatMajorVersion func() FormatMajorVersion,
 	mu *sync.Mutex,
 ) error {
-	vs.init(dirname, opts, marker, setCurrent, mu)
+	vs.init(dirname, opts, marker, setCurrent, getFormatMajorVersion, mu)
 	newVersion := &version{}
 	vs.append(newVersion)
 	var err error
@@ -199,9 +208,10 @@ func (vs *versionSet) load(
 	manifestFileNum FileNum,
 	marker *atomicfs.Marker,
 	setCurrent func(FileNum) error,
+	getFormatMajorVersion func() FormatMajorVersion,
 	mu *sync.Mutex,
 ) error {
-	vs.init(dirname, opts, marker, setCurrent, mu)
+	vs.init(dirname, opts, marker, setCurrent, getFormatMajorVersion, mu)
 
 	vs.manifestFileNum = manifestFileNum
 	manifestPath := base.MakeFilepath(opts.FS, dirname, fileTypeManifest, vs.manifestFileNum.DiskFileNum())
@@ -290,16 +300,17 @@ func (vs *versionSet) load(
 	// Populate the fileBackingMap and the FileBacking for virtual sstables since
 	// we have finished version edit accumulation.
 	for _, s := range bve.AddedFileBacking {
-		vs.fileBackingMap[s.DiskFileNum] = s
+		vs.addFileBacking(s)
 	}
 
 	for _, fileNum := range bve.RemovedFileBacking {
-		delete(vs.fileBackingMap, fileNum)
+		vs.removeFileBacking(fileNum)
 	}
 
 	newVersion, err := bve.Apply(
 		nil, vs.cmp, opts.Comparer.FormatKey, opts.FlushSplitBytes,
 		opts.Experimental.ReadCompactionRate, nil, /* zombies */
+		getFormatMajorVersion().orderingInvariants(),
 	)
 	if err != nil {
 		return err
@@ -357,6 +368,26 @@ func (vs *versionSet) logUnlock() {
 	vs.writerCond.Signal()
 }
 
+// Only call if the DiskFileNum doesn't exist in the fileBackingMap.
+func (vs *versionSet) addFileBacking(backing *manifest.FileBacking) {
+	_, ok := vs.backingState.fileBackingMap[backing.DiskFileNum]
+	if ok {
+		panic("pebble: trying to add an existing file backing")
+	}
+	vs.backingState.fileBackingMap[backing.DiskFileNum] = backing
+	vs.backingState.fileBackingSize += backing.Size
+}
+
+// Only call if the the DiskFileNum exists in the fileBackingMap.
+func (vs *versionSet) removeFileBacking(dfn base.DiskFileNum) {
+	backing, ok := vs.backingState.fileBackingMap[dfn]
+	if !ok {
+		panic("pebble: trying to remove an unknown file backing")
+	}
+	delete(vs.backingState.fileBackingMap, dfn)
+	vs.backingState.fileBackingSize -= backing.Size
+}
+
 // logAndApply logs the version edit to the manifest, applies the version edit
 // to the current version, and installs the new version.
 //
@@ -414,6 +445,8 @@ func (vs *versionSet) logAndApply(
 	}
 
 	currentVersion := vs.currentVersion()
+	fmv := vs.getFormatMajorVersion()
+	orderingInvariants := fmv.orderingInvariants()
 	var newVersion *version
 
 	// Generate a new manifest if we don't currently have one, or forceRotation
@@ -484,13 +517,14 @@ func (vs *versionSet) logAndApply(
 		defer vs.mu.Lock()
 
 		var err error
-		if vs.opts.FormatMajorVersion < ExperimentalFormatVirtualSSTables && len(ve.CreatedBackingTables) > 0 {
+		if vs.getFormatMajorVersion() < FormatVirtualSSTables && len(ve.CreatedBackingTables) > 0 {
 			return errors.AssertionFailedf("MANIFEST cannot contain virtual sstable records due to format major version")
 		}
 		newVersion, zombies, err = manifest.AccumulateIncompleteAndApplySingleVE(
 			ve, currentVersion, vs.cmp, vs.opts.Comparer.FormatKey,
 			vs.opts.FlushSplitBytes, vs.opts.Experimental.ReadCompactionRate,
-			vs.fileBackingMap,
+			vs.backingState.fileBackingMap, vs.addFileBacking, vs.removeFileBacking,
+			orderingInvariants,
 		)
 		if err != nil {
 			return errors.Wrap(err, "MANIFEST apply failed")
@@ -588,6 +622,8 @@ func (vs *versionSet) logAndApply(
 	for i := range vs.metrics.Levels {
 		l := &vs.metrics.Levels[i]
 		l.NumFiles = int64(newVersion.Levels[i].Len())
+		l.NumVirtualFiles = newVersion.Levels[i].NumVirtual
+		l.VirtualSize = newVersion.Levels[i].VirtualSize
 		l.Size = int64(newVersion.Levels[i].Size())
 
 		l.Sublevels = 0
@@ -595,13 +631,22 @@ func (vs *versionSet) logAndApply(
 			l.Sublevels = 1
 		}
 		if invariants.Enabled {
-			if count := int64(newVersion.Levels[i].Len()); l.NumFiles != count {
-				vs.opts.Logger.Fatalf("versionSet metrics L%d NumFiles = %d, actual count = %d", i, l.NumFiles, count)
-			}
 			levelFiles := newVersion.Levels[i].Slice()
 			if size := int64(levelFiles.SizeSum()); l.Size != size {
 				vs.opts.Logger.Fatalf("versionSet metrics L%d Size = %d, actual size = %d", i, l.Size, size)
 			}
+			if nVirtual := levelFiles.NumVirtual(); nVirtual != l.NumVirtualFiles {
+				vs.opts.Logger.Fatalf(
+					"versionSet metrics L%d NumVirtual = %d, actual NumVirtual = %d",
+					i, l.NumVirtualFiles, nVirtual,
+				)
+			}
+			if vSize := levelFiles.VirtualSizeSum(); vSize != l.VirtualSize {
+				vs.opts.Logger.Fatalf(
+					"versionSet metrics L%d Virtual size = %d, actual size = %d",
+					i, l.VirtualSize, vSize,
+				)
+			}
 		}
 	}
 	vs.metrics.Levels[0].Sublevels = int32(len(newVersion.L0SublevelFiles))
@@ -613,7 +658,9 @@ func (vs *versionSet) logAndApply(
 	return nil
 }
 
-func (vs *versionSet) incrementCompactions(kind compactionKind, extraLevels []*compactionLevel) {
+func (vs *versionSet) incrementCompactions(
+	kind compactionKind, extraLevels []*compactionLevel, pickerMetrics compactionPickerMetrics,
+) {
 	switch kind {
 	case compactionKindDefault:
 		vs.metrics.Compact.Count++

vendor/github.com/cockroachdb/pebble/vfs/mem_fs.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"syscall"
 	"time"
 
 	"github.com/cockroachdb/errors"
@@ -78,6 +79,11 @@ type MemFS struct {
 	mu   sync.Mutex
 	root *memNode
 
+	// lockFiles holds a map of open file locks. Presence in this map indicates
+	// a file lock is currently held. Keys are strings holding the path of the
+	// locked file. The stored value is untyped and  unused; only presence of
+	// the key within the map is significant.
+	lockedFiles sync.Map
 	strict      bool
 	ignoreSyncs bool
 	// Windows has peculiar semantics with respect to hard links and deleting
@@ -457,9 +463,31 @@ func (y *MemFS) MkdirAll(dirname string, perm os.FileMode) error {
 // Lock implements FS.Lock.
 func (y *MemFS) Lock(fullname string) (io.Closer, error) {
 	// FS.Lock excludes other processes, but other processes cannot see this
-	// process' memory. We translate Lock into Create so that have the normal
+	// process' memory. However some uses (eg, Cockroach tests) may open and
-	// detection of non-existent directory paths.
+	// close the same MemFS-backed database multiple times. We want mutual
-	return y.Create(fullname)
+	// exclusion in this case too. See cockroachdb/cockroach#110645.
+	_, loaded := y.lockedFiles.Swap(fullname, nil /* the value itself is insignificant */)
+	if loaded {
+		// This file lock has already been acquired. On unix, this results in
+		// either EACCES or EAGAIN so we mimic.
+		return nil, syscall.EAGAIN
+	}
+	// Otherwise, we successfully acquired the lock. Locks are visible in the
+	// parent directory listing, and they also must be created under an existent
+	// directory. Create the path so that we have the normal detection of
+	// non-existent directory paths, and make the lock visible when listing
+	// directory entries.
+	f, err := y.Create(fullname)
+	if err != nil {
+		// "Release" the lock since we failed.
+		y.lockedFiles.Delete(fullname)
+		return nil, err
+	}
+	return &memFileLock{
+		y:        y,
+		f:        f,
+		fullname: fullname,
+	}, nil
 }
 
 // List implements FS.List.
@@ -787,3 +815,18 @@ func (f *memFile) Fd() uintptr {
 func (f *memFile) Flush() error {
 	return nil
 }
+
+type memFileLock struct {
+	y        *MemFS
+	f        File
+	fullname string
+}
+
+func (l *memFileLock) Close() error {
+	if l.y == nil {
+		return nil
+	}
+	l.y.lockedFiles.Delete(l.fullname)
+	l.y = nil
+	return l.f.Close()
+}

vendor/github.com/eleksir/go-xmpp/.golangci.yml

@@ -0,0 +1,66 @@
+run:
+  tests: false
+  skip-dirs:
+    - _example
+  modules-download-mode: readonly
+  allow-parallel-runners: true
+  go: '1.21'
+output:
+  sort-results: true
+linters-settings:
+  gosec:
+    excludes:
+      # Оно не используется для повышения безопасности или каких-то таких секъюрых вещей, оно используется для выборки
+      # произвольного элемента из массива, так что ИДИ НАХУЙ с этой проверкой и не еби мозги липовой безопасностью.
+      - G404 # Insecure random number source (rand)
+linters:
+  disable-all: true
+  enable:
+    - errcheck
+    - gosimple
+    - govet
+    - ineffassign
+    - staticcheck
+    - unused
+    - asasalint
+    - asciicheck
+    - bodyclose
+    - containedctx
+    - contextcheck
+    - decorder
+    - durationcheck
+    - errchkjson
+    - errorlint
+    - exhaustive
+    - exportloopref
+    - gci
+    - ginkgolinter
+    - gocheckcompilerdirectives
+    - gocritic
+    - godot
+    - gofmt
+    - goimports
+    - gosec
+    - gosmopolitan
+    - grouper
+    - importas
+    - ireturn
+    - makezero
+    - mirror
+    - nakedret
+    - nilerr
+    - nilnil
+    - nlreturn
+    - noctx
+    - nolintlint
+    - nonamedreturns
+    - predeclared
+    - promlinter
+    - reassign
+    - revive
+    - rowserrcheck
+    - stylecheck
+    - unconvert
+    - wastedassign
+    - whitespace
+    - wsl

vendor/github.com/eleksir/go-xmpp/README.md

@@ -3,12 +3,12 @@ go-xmpp
 
 go xmpp library.
 
-[Original](https://github.com/rsc/rsc/blob/master/xmpp/xmpp.go) was done by [Russ Cox](https://github.com/rsc), then
+[Original](https://github.com/rsc/rsc/blob/master/xmpp/xmpp.go) work done by [Russ Cox](https://github.com/rsc), 
-forked by [Yasuhiro Matsumoto](https://github.com/mattn).
+then [Yasuhiro Matsumoto](https://github.com/mattn) fork it and continue to improve it.
 
-Since both of them have amost zero interest in continuing develop of this lib, i decide to fork it.
+Since both of them have almost zero interest to continue develop this lib, I decide to fork it.
 
 Documentation can be found at [godoc project](https://godoc.org/github.com/eleksir/go-xmpp)
 
-This fork mainains my needs for xmpp protocol. Feel free to contribute via PR-s if you find it interesing too and feel
+This fork maintains my needs for xmpp protocol. Feel free to contribute via PR-s if you find it interesting too and feel
-that it needs more love than it already has.
+that it needs more love than it already has. And of course fell free to for it if needed.

vendor/github.com/eleksir/go-xmpp/xmpp_avatar.go

@@ -1,7 +1,7 @@
 package xmpp
 
 import (
-	"crypto/sha1"
+	"crypto/sha1" //nolint:gosec // go fuck yourself, gosec. You don't even fucking read rfc. So shout the fuck up.
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/xml"
@@ -10,8 +10,8 @@ import (
 )
 
 const (
-	XMPPNS_AVATAR_PEP_DATA     = "urn:xmpp:avatar:data"
+	XMPPNS_AVATAR_PEP_DATA     = "urn:xmpp:avatar:data"     //nocritic:revive
-	XMPPNS_AVATAR_PEP_METADATA = "urn:xmpp:avatar:metadata"
+	XMPPNS_AVATAR_PEP_METADATA = "urn:xmpp:avatar:metadata" //nocritic:revive
 )
 
 type clientAvatarData struct {
@@ -52,8 +52,8 @@ type AvatarMetadata struct {
 
 func handleAvatarData(itemsBody []byte, from, id string) (AvatarData, error) {
 	var data clientAvatarData
-	err := xml.Unmarshal(itemsBody, &data)
+
-	if err != nil {
+	if err := xml.Unmarshal(itemsBody, &data); err != nil {
 		return AvatarData{}, err
 	}
 
@@ -64,8 +64,9 @@ func handleAvatarData(itemsBody []byte, from, id string) (AvatarData, error) {
 		return AvatarData{}, err
 	}
 
-	hash := sha1.Sum(dataRaw)
+	hash := sha1.Sum(dataRaw) //nolint:gosec // go fuck yourself, gosec. This exact mechanics described in standard.
 	hashStr := hex.EncodeToString(hash[:])
+
 	if hashStr != id {
 		return AvatarData{}, errors.New("SHA1 hashes do not match")
 	}
@@ -77,9 +78,12 @@ func handleAvatarData(itemsBody []byte, from, id string) (AvatarData, error) {
 }
 
 func handleAvatarMetadata(body []byte, from string) (AvatarMetadata, error) {
-	var meta clientAvatarMetadata
+	var (
-	err := xml.Unmarshal(body, &meta)
+		meta clientAvatarMetadata
-	if err != nil {
+		err  error
+	)
+
+	if err = xml.Unmarshal(body, &meta); err != nil {
 		return AvatarMetadata{}, err
 	}
 
@@ -91,10 +95,10 @@ func handleAvatarMetadata(body []byte, from string) (AvatarMetadata, error) {
 		ID:     meta.Info.ID,
 		Type:   meta.Info.Type,
 		URL:    meta.Info.URL,
-	}, nil
+	}, err
 }
 
-// A wrapper for atoi which just returns -1 if an error occurs
+// atoiw is a wrapper for atoi which just returns -1 if an error occurs.
 func atoiw(str string) int {
 	i, err := strconv.Atoi(str)
 	if err != nil {
@@ -104,22 +108,22 @@ func atoiw(str string) int {
 	return i
 }
 
-func (c *Client) AvatarSubscribeMetadata(jid string) {
+func (c *Client) AvatarSubscribeMetadata(jid string) (string, error) {
-	c.PubsubSubscribeNode(XMPPNS_AVATAR_PEP_METADATA, jid)
+	return c.PubsubSubscribeNode(XMPPNS_AVATAR_PEP_METADATA, jid)
 }
 
-func (c *Client) AvatarUnsubscribeMetadata(jid string) {
+func (c *Client) AvatarUnsubscribeMetadata(jid string) (string, error) {
-	c.PubsubUnsubscribeNode(XMPPNS_AVATAR_PEP_METADATA, jid)
+	return c.PubsubUnsubscribeNode(XMPPNS_AVATAR_PEP_METADATA, jid)
 }
 
-func (c *Client) AvatarRequestData(jid string) {
+func (c *Client) AvatarRequestData(jid string) (string, error) {
-	c.PubsubRequestLastItems(XMPPNS_AVATAR_PEP_DATA, jid)
+	return c.PubsubRequestLastItems(XMPPNS_AVATAR_PEP_DATA, jid)
 }
 
-func (c *Client) AvatarRequestDataByID(jid, id string) {
+func (c *Client) AvatarRequestDataByID(jid, id string) (string, error) {
-	c.PubsubRequestItem(XMPPNS_AVATAR_PEP_DATA, jid, id)
+	return c.PubsubRequestItem(XMPPNS_AVATAR_PEP_DATA, jid, id)
 }
 
-func (c *Client) AvatarRequestMetadata(jid string) {
+func (c *Client) AvatarRequestMetadata(jid string) (string, error) {
-	c.PubsubRequestLastItems(XMPPNS_AVATAR_PEP_METADATA, jid)
+	return c.PubsubRequestLastItems(XMPPNS_AVATAR_PEP_METADATA, jid)
 }

vendor/github.com/eleksir/go-xmpp/xmpp_disco.go

@@ -5,8 +5,8 @@ import (
 )
 
 const (
-	XMPPNS_DISCO_ITEMS = "http://jabber.org/protocol/disco#items"
+	XMPPNS_DISCO_ITEMS = "http://jabber.org/protocol/disco#items" //nocritic:revive
-	XMPPNS_DISCO_INFO  = "http://jabber.org/protocol/disco#info"
+	XMPPNS_DISCO_INFO  = "http://jabber.org/protocol/disco#info"  //nocritic:revive
 )
 
 type clientDiscoFeature struct {
@@ -90,6 +90,7 @@ func clientIdentitiesToReturn(identities []clientDiscoIdentity) []DiscoIdentity
 
 func clientDiscoItemsToReturn(items []clientDiscoItem) []DiscoItem {
 	var ret []DiscoItem
+
 	for _, item := range items {
 		ret = append(ret, DiscoItem{
 			Jid:  item.Jid,

vendor/github.com/eleksir/go-xmpp/xmpp_information_query.go

@@ -14,22 +14,25 @@ const IQTypeError = "error"
 func (c *Client) Discovery() (string, error) {
 	// use getCookie for a pseudo random id.
 	reqID := strconv.FormatUint(uint64(getCookie()), 10)
+
 	return c.RawInformationQuery(c.jid, c.domain, reqID, IQTypeGet, XMPPNS_DISCO_ITEMS, "")
 }
 
 // DiscoverNodeInfo discovers information about a node. Empty node queries info about server itself.
 func (c *Client) DiscoverNodeInfo(node string) (string, error) {
 	query := fmt.Sprintf("<query xmlns='%s' node='%s'/>", XMPPNS_DISCO_INFO, node)
+
 	return c.RawInformation(c.jid, c.domain, "info3", IQTypeGet, query)
 }
 
 // DiscoverInfo discovers information about given item from given jid.
 func (c *Client) DiscoverInfo(from string, to string) (string, error) {
 	query := fmt.Sprintf("<query xmlns='%s'/>", XMPPNS_DISCO_INFO)
+
 	return c.RawInformation(from, to, "info3", IQTypeGet, query)
 }
 
-// DiscoverServerItems discover items that the server exposes
+// DiscoverServerItems discover items that the server exposes.
 func (c *Client) DiscoverServerItems() (string, error) {
 	return c.DiscoverEntityItems(c.domain)
 }
@@ -37,6 +40,7 @@ func (c *Client) DiscoverServerItems() (string, error) {
 // DiscoverEntityItems discovers items that an entity exposes.
 func (c *Client) DiscoverEntityItems(jid string) (string, error) {
 	query := fmt.Sprintf("<query xmlns='%s'/>", XMPPNS_DISCO_ITEMS)
+
 	return c.RawInformation(c.jid, jid, "info1", IQTypeGet, query)
 }
 
@@ -44,6 +48,7 @@ func (c *Client) DiscoverEntityItems(jid string) (string, error) {
 func (c *Client) RawInformationQuery(from, to, id, iqType, requestNamespace, body string) (string, error) {
 	const xmlIQ = "<iq from='%s' to='%s' id='%s' type='%s'><query xmlns='%s'>%s</query></iq>"
 	_, err := fmt.Fprintf(StanzaWriter, xmlIQ, xmlEscape(from), xmlEscape(to), id, iqType, requestNamespace, body)
+
 	return id, err
 }
 
@@ -51,6 +56,7 @@ func (c *Client) RawInformationQuery(from, to, id, iqType, requestNamespace, bod
 func (c *Client) RawInformation(from, to, id, iqType, body string) (string, error) {
 	const xmlIQ = "<iq from='%s' to='%s' id='%s' type='%s'>%s</iq>"
 	_, err := fmt.Fprintf(StanzaWriter, xmlIQ, xmlEscape(from), xmlEscape(to), id, iqType, body)
+
 	return id, err
 }
 
@@ -64,7 +70,7 @@ func (c *Client) IqVersionResponse(v IQ, name, version, os string) (string, erro
 		version = "undefined"
 	}
 
-	query := "<query xmlns=\"jabber:iq:version\">" //nolint:wsl
+	query := "<query xmlns=\"jabber:iq:version\">"
 	query += fmt.Sprintf("<name>%s</name>", name)
 	query += fmt.Sprintf("<version>%s</version>", version)
 
@@ -100,9 +106,9 @@ func (c *Client) JabberIqLastResponse(v IQ, lastActivity int64) (string, error)
 	)
 }
 
-// UrnXmppTimeResponse implements response to query entity's current time (xep-0202).
+// UrnXMPPTimeResponse implements response to query entity's current time (xep-0202).
-// TimezoneOffset have format +HH:MM or -HH:MM
+// TimezoneOffset have format +HH:MM or -HH:MM.
-func (c *Client) UrnXmppTimeResponse(v IQ, timezoneOffset string) (string, error) {
+func (c *Client) UrnXMPPTimeResponse(v IQ, timezoneOffset string) (string, error) {
 	query := fmt.Sprintf(
 		"<time xmlns=\"urn:xmpp:time\"><tzo>%s</tzo><utc>%s</utc></time>",
 		timezoneOffset,

vendor/github.com/eleksir/go-xmpp/xmpp_muc.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 // TODO(flo):
-//   - support password protected MUC rooms
 //   - cleanup signatures of join/leave functions
 package xmpp
 
@@ -23,16 +22,18 @@ const (
 	SinceHistory   = 4
 )
 
-// Send sends room topic wrapped inside an XMPP message stanza body.
+// SendTopic sends room topic wrapped inside an XMPP message stanza body.
-func (c *Client) SendTopic(chat Chat) (n int, err error) {
+func (c *Client) SendTopic(chat Chat) (int, error) {
 	return fmt.Fprintf(StanzaWriter, "<message to='%s' type='%s' xml:lang='en'>"+"<subject>%s</subject></message>",
 		xmlEscape(chat.Remote), xmlEscape(chat.Type), xmlEscape(chat.Text))
 }
 
-func (c *Client) JoinMUCNoHistory(jid, nick string) (n int, err error) {
+// JoinMUCNoHistory joins room and instructs server that no history data required.
+func (c *Client) JoinMUCNoHistory(jid, nick string) (int, error) {
 	if nick == "" {
 		nick = c.jid
 	}
+
 	return fmt.Fprintf(StanzaWriter, "<presence to='%s/%s'>\n"+
 		"<x xmlns='%s'>"+
 		"<history maxchars='0'/></x>\n"+
@@ -40,12 +41,13 @@ func (c *Client) JoinMUCNoHistory(jid, nick string) (n int, err error) {
 		xmlEscape(jid), xmlEscape(nick), nsMUC)
 }
 
-// xep-0045 7.2
+// JoinMUC joins room, xep-0045 7.2.
-func (c *Client) JoinMUC(jid, nick string, history_type, history int, history_date *time.Time) (n int, err error) {
+func (c *Client) JoinMUC(jid, nick string, historyType, history int, historyDate *time.Time) (int, error) {
 	if nick == "" {
 		nick = c.jid
 	}
-	switch history_type {
+
+	switch historyType {
 	case NoHistory:
 		return fmt.Fprintf(StanzaWriter, "<presence to='%s/%s'>\n"+
 			"<x xmlns='%s' />\n"+
@@ -70,23 +72,25 @@ func (c *Client) JoinMUC(jid, nick string, history_type, history int, history_da
 			"</presence>",
 			xmlEscape(jid), xmlEscape(nick), nsMUC, history)
 	case SinceHistory:
-		if history_date != nil {
+		if historyDate != nil {
 			return fmt.Fprintf(StanzaWriter, "<presence to='%s/%s'>\n"+
 				"<x xmlns='%s'>\n"+
 				"<history since='%s'/></x>\n"+
 				"</presence>",
-				xmlEscape(jid), xmlEscape(nick), nsMUC, history_date.Format(time.RFC3339))
+				xmlEscape(jid), xmlEscape(nick), nsMUC, historyDate.Format(time.RFC3339))
 		}
 	}
-	return 0, errors.New("Unknown history option")
+
+	return 0, errors.New("unknown history option")
 }
 
-// xep-0045 7.2.6
+// JoinProtectedMUC joins password protected room, xep-0045 7.2.6.
-func (c *Client) JoinProtectedMUC(jid, nick string, password string, history_type, history int, history_date *time.Time) (n int, err error) {
+func (c *Client) JoinProtectedMUC(jid, nick string, password string, historyType, history int, historyDate *time.Time) (int, error) {
 	if nick == "" {
 		nick = c.jid
 	}
-	switch history_type {
+
+	switch historyType {
 	case NoHistory:
 		return fmt.Fprintf(StanzaWriter, "<presence to='%s/%s'>\n"+
 			"<x xmlns='%s'>\n"+
@@ -116,20 +120,21 @@ func (c *Client) JoinProtectedMUC(jid, nick string, password string, history_typ
 			"</presence>",
 			xmlEscape(jid), xmlEscape(nick), nsMUC, xmlEscape(password), history)
 	case SinceHistory:
-		if history_date != nil {
+		if historyDate != nil {
 			return fmt.Fprintf(StanzaWriter, "<presence to='%s/%s'>\n"+
 				"<x xmlns='%s'>\n"+
 				"<password>%s</password>\n"+
 				"<history since='%s'/></x>\n"+
 				"</presence>",
-				xmlEscape(jid), xmlEscape(nick), nsMUC, xmlEscape(password), history_date.Format(time.RFC3339))
+				xmlEscape(jid), xmlEscape(nick), nsMUC, xmlEscape(password), historyDate.Format(time.RFC3339))
 		}
 	}
-	return 0, errors.New("Unknown history option")
+
+	return 0, errors.New("unknown history option")
 }
 
-// xep-0045 7.14
+// LeaveMUC quits from room xep-0045 7.14.
-func (c *Client) LeaveMUC(jid string) (n int, err error) {
+func (c *Client) LeaveMUC(jid string) (int, error) {
 	return fmt.Fprintf(StanzaWriter, "<presence from='%s' to='%s' type='unavailable' />",
 		c.jid, xmlEscape(jid))
 }

vendor/github.com/eleksir/go-xmpp/xmpp_ping.go

@@ -8,13 +8,16 @@ func (c *Client) PingC2S(jid, server string) error {
 	if jid == "" {
 		jid = c.jid
 	}
+
 	if server == "" {
 		server = c.domain
 	}
+
 	_, err := fmt.Fprintf(StanzaWriter, "<iq from='%s' to='%s' id='c2s1' type='get'>\n"+
 		"<ping xmlns='urn:xmpp:ping'/>\n"+
 		"</iq>",
 		xmlEscape(jid), xmlEscape(server))
+
 	return err
 }
 
@@ -23,16 +26,18 @@ func (c *Client) PingS2S(fromServer, toServer string) error {
 		"<ping xmlns='urn:xmpp:ping'/>\n"+
 		"</iq>",
 		xmlEscape(fromServer), xmlEscape(toServer))
+
 	return err
 }
 
 func (c *Client) SendResultPing(id, toServer string) error {
 	_, err := fmt.Fprintf(StanzaWriter, "<iq type='result' to='%s' id='%s'/>",
 		xmlEscape(toServer), xmlEscape(id))
+
 	return err
 }
 
-// PingResponse responding to ping query according to xep-0199
+// PingResponse responding to ping query according to xep-0199.
 func (c *Client) PingResponse(v IQ) (string, error) {
 	return c.RawInformation(
 		v.To,

vendor/github.com/eleksir/go-xmpp/xmpp_pubsub.go

@@ -6,8 +6,8 @@ import (
 )
 
 const (
-	XMPPNS_PUBSUB       = "http://jabber.org/protocol/pubsub"
+	XMPPNS_PUBSUB       = "http://jabber.org/protocol/pubsub"       //nolint:revive
-	XMPPNS_PUBSUB_EVENT = "http://jabber.org/protocol/pubsub#event"
+	XMPPNS_PUBSUB_EVENT = "http://jabber.org/protocol/pubsub#event" //nolint:revive
 )
 
 type clientPubsubItem struct {
@@ -67,7 +67,7 @@ type PubsubItems struct {
 	Items []PubsubItem
 }
 
-// Converts []clientPubsubItem to []PubsubItem
+// Converts []clientPubsubItem to []PubsubItem.
 func pubsubItemsToReturn(items []clientPubsubItem) []PubsubItem {
 	var tmp []PubsubItem
 	for _, i := range items {
@@ -96,6 +96,7 @@ func pubsubSubscriptionStanza(node, jid string) string {
 	body := fmt.Sprintf("<subscribe node='%s' jid='%s'/>",
 		xmlEscape(node),
 		xmlEscape(jid))
+
 	return pubsubStanza(body)
 }
 
@@ -103,6 +104,7 @@ func pubsubUnsubscriptionStanza(node, jid string) string {
 	body := fmt.Sprintf("<unsubscribe node='%s' jid='%s'/>",
 		xmlEscape(node),
 		xmlEscape(jid))
+
 	return pubsubStanza(body)
 }
 
@@ -124,10 +126,12 @@ func (c *Client) PubsubUnsubscribeNode(node, jid string) (string, error) {
 
 func (c *Client) PubsubRequestLastItems(node, jid string) (string, error) {
 	body := fmt.Sprintf("<items node='%s'/>", node)
+
 	return c.RawInformation(c.jid, jid, "items1", "get", pubsubStanza(body))
 }
 
 func (c *Client) PubsubRequestItem(node, jid, id string) (string, error) {
 	body := fmt.Sprintf("<items node='%s'><item id='%s'/></items>", node, id)
+
 	return c.RawInformation(c.jid, jid, "items3", "get", pubsubStanza(body))
 }

vendor/github.com/eleksir/go-xmpp/xmpp_subscription.go

@@ -4,22 +4,30 @@ import (
 	"fmt"
 )
 
-func (c *Client) ApproveSubscription(jid string) {
+func (c *Client) ApproveSubscription(jid string) error {
-	fmt.Fprintf(StanzaWriter, "<presence to='%s' type='subscribed'/>",
+	_, err := fmt.Fprintf(StanzaWriter, "<presence to='%s' type='subscribed'/>",
 		xmlEscape(jid))
+
+	return err
 }
 
-func (c *Client) RevokeSubscription(jid string) {
+func (c *Client) RevokeSubscription(jid string) error {
-	fmt.Fprintf(StanzaWriter, "<presence to='%s' type='unsubscribed'/>",
+	_, err := fmt.Fprintf(StanzaWriter, "<presence to='%s' type='unsubscribed'/>",
 		xmlEscape(jid))
+
+	return err
 }
 
-func (c *Client) RetrieveSubscription(jid string) {
+func (c *Client) RetrieveSubscription(jid string) error {
-	fmt.Fprintf(c.conn, "<presence to='%s' type='unsubscribe'/>",
+	_, err := fmt.Fprintf(c.conn, "<presence to='%s' type='unsubscribe'/>",
 		xmlEscape(jid))
+
+	return err
 }
 
-func (c *Client) RequestSubscription(jid string) {
+func (c *Client) RequestSubscription(jid string) error {
-	fmt.Fprintf(StanzaWriter, "<presence to='%s' type='subscribe'/>",
+	_, err := fmt.Fprintf(StanzaWriter, "<presence to='%s' type='subscribe'/>",
 		xmlEscape(jid))
+
+	return err
 }

vendor/github.com/getsentry/sentry-go/.golangci.yml

@@ -2,8 +2,6 @@ linters:
   disable-all: true
   enable:
     - bodyclose
-    - deadcode
-    - depguard
     - dogsled
     - dupl
     - errcheck
@@ -24,12 +22,10 @@ linters:
     - prealloc
     - revive
     - staticcheck
-    - structcheck
     - typecheck
     - unconvert
     - unparam
     - unused
-    - varcheck
     - whitespace
 issues:
   exclude-rules:

vendor/github.com/getsentry/sentry-go/CHANGELOG.md

@@ -1,9 +1,87 @@
 # Changelog
 
+## 0.27.0
+
+The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.27.0.
+
+### Breaking Changes
+
+- `Exception.ThreadId` is now typed as `uint64`. It was wrongly typed as `string` before. ([#770](https://github.com/getsentry/sentry-go/pull/770))
+
+### Misc
+
+- Export `Event.Attachments` ([#771](https://github.com/getsentry/sentry-go/pull/771))
+
+## 0.26.0
+
+The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.26.0.
+
+### Breaking Changes
+
+As previously announced, this release removes some methods from the SDK.
+
+- `sentry.TransactionName()` use `sentry.WithTransactionName()` instead.
+- `sentry.OpName()` use `sentry.WithOpName()` instead.
+- `sentry.TransctionSource()` use `sentry.WithTransactionSource()` instead.
+- `sentry.SpanSampled()` use `sentry.WithSpanSampled()` instead.
+
+### Features
+
+- Add `WithDescription` span option ([#751](https://github.com/getsentry/sentry-go/pull/751))
+
+  ```go
+  span := sentry.StartSpan(ctx, "http.client", WithDescription("GET /api/users"))
+  ```
+- Add support for package name parsing in Go 1.20 and higher ([#730](https://github.com/getsentry/sentry-go/pull/730))
+
+### Bug Fixes
+
+- Apply `ClientOptions.SampleRate` only to errors & messages ([#754](https://github.com/getsentry/sentry-go/pull/754))
+- Check if git is available before executing any git commands ([#737](https://github.com/getsentry/sentry-go/pull/737))
+
+## 0.25.0
+
+The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.25.0.
+
+### Breaking Changes
+
+As previously announced, this release removes two global constants from the SDK.
+
+- `sentry.Version` was removed. Use `sentry.SDKVersion` instead ([#727](https://github.com/getsentry/sentry-go/pull/727))
+- `sentry.SDKIdentifier` was removed. Use `Client.GetSDKIdentifier()` instead ([#727](https://github.com/getsentry/sentry-go/pull/727))
+
+### Features
+
+- Add `ClientOptions.IgnoreTransactions`, which allows you to ignore specific transactions based on their name ([#717](https://github.com/getsentry/sentry-go/pull/717))
+- Add `ClientOptions.Tags`, which allows you to set global tags that are applied to all events. You can also define tags by setting `SENTRY_TAGS_` environment variables ([#718](https://github.com/getsentry/sentry-go/pull/718))
+
+### Bug fixes
+
+- Fix an issue in the profiler that would cause an infinite loop if the duration of a transaction is longer than 30 seconds ([#724](https://github.com/getsentry/sentry-go/issues/724))
+
+### Misc
+
+- `dsn.RequestHeaders()` is not to be removed, though it is still considered deprecated and should only be used when using a custom transport that sends events to the `/store` endpoint ([#720](https://github.com/getsentry/sentry-go/pull/720))
+
+## 0.24.1
+
+The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.24.1.
+
+### Bug fixes
+
+- Prevent a panic in `sentryotel.flushSpanProcessor()` ([(#711)](https://github.com/getsentry/sentry-go/pull/711))
+- Prevent a panic when setting the SDK identifier ([#715](https://github.com/getsentry/sentry-go/pull/715))
+
 ## 0.24.0
 
 The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.24.0.
 
+### Deprecations
+
+- `sentry.Version` to be removed in 0.25.0. Use `sentry.SDKVersion` instead.
+- `sentry.SDKIdentifier` to be removed in 0.25.0. Use `Client.GetSDKIdentifier()` instead.
+- `dsn.RequestHeaders()` to be removed after 0.25.0, but no earlier than December 1, 2023. Requests to the `/envelope` endpoint are authenticated using the DSN in the envelope header.
+
 ### Features
 
 - Run a single instance of the profiler instead of multiple ones for each Go routine ([#655](https://github.com/getsentry/sentry-go/pull/655))

vendor/github.com/getsentry/sentry-go/client.go

@@ -140,6 +140,9 @@ type ClientOptions struct {
 	// and if applicable, caught errors type and value.
 	// If the match is found, then a whole event will be dropped.
 	IgnoreErrors []string
+	// List of regexp strings that will be used to match against a transaction's
+	// name.  If a match is found, then the transaction  will be dropped.
+	IgnoreTransactions []string
 	// If this flag is enabled, certain personally identifiable information (PII) is added by active integrations.
 	// By default, no such data is sent.
 	SendDefaultPII bool
@@ -221,6 +224,8 @@ type ClientOptions struct {
 	// is not optimized for long chains either. The top-level error together with a
 	// stack trace is often the most useful information.
 	MaxErrorDepth int
+	// Default event tags. These are overridden by tags set on a scope.
+	Tags map[string]string
 }
 
 // Client is the underlying processor that is used by the main API and Hub
@@ -371,6 +376,8 @@ func (client *Client) setupIntegrations() {
 		new(environmentIntegration),
 		new(modulesIntegration),
 		new(ignoreErrorsIntegration),
+		new(ignoreTransactionsIntegration),
+		new(globalTagsIntegration),
 	}
 
 	if client.options.Integrations != nil {
@@ -598,9 +605,9 @@ func (client *Client) processEvent(event *Event, hint *EventHint, scope EventMod
 	}
 
 	// Transactions are sampled by options.TracesSampleRate or
-	// options.TracesSampler when they are started. All other events
+	// options.TracesSampler when they are started. Other events
-	// (errors, messages) are sampled here.
+	// (errors, messages) are sampled here. Does not apply to check-ins.
-	if event.Type != transactionType && !sample(client.options.SampleRate) {
+	if event.Type != transactionType && event.Type != checkInType && !sample(client.options.SampleRate) {
 		Logger.Println("Event dropped due to SampleRate hit.")
 		return nil
 	}
@@ -619,7 +626,7 @@ func (client *Client) processEvent(event *Event, hint *EventHint, scope EventMod
 			Logger.Println("Transaction dropped due to BeforeSendTransaction callback.")
 			return nil
 		}
-	} else if event.Type != transactionType && client.options.BeforeSend != nil {
+	} else if event.Type != transactionType && event.Type != checkInType && client.options.BeforeSend != nil {
 		// All other events
 		if event = client.options.BeforeSend(event, hint); event == nil {
 			Logger.Println("Event dropped due to BeforeSend callback.")

vendor/github.com/getsentry/sentry-go/dsn.go

@@ -196,11 +196,15 @@ func (dsn Dsn) GetAPIURL() *url.URL {
 	return parsedURL
 }
 
-// RequestHeaders returns all the necessary headers that have to be used in the transport.
+// RequestHeaders returns all the necessary headers that have to be used in the transport when seinding events
-// Deprecated: To be removed after 0.25.0, but no earlier than December 1, 2023. Requests to /envelope are authenticated using the DSN in the envelope header itself.
+// to the /store endpoint.
+//
+// Deprecated: This method shall only be used if you want to implement your own transport that sends events to
+// the /store endpoint. If you're using the transport provided by the SDK, all necessary headers to authenticate
+// against the /envelope endpoint are added automatically.
 func (dsn Dsn) RequestHeaders() map[string]string {
 	auth := fmt.Sprintf("Sentry sentry_version=%s, sentry_timestamp=%d, "+
-		"sentry_client=sentry.go/%s, sentry_key=%s", apiVersion, time.Now().Unix(), Version, dsn.publicKey)
+		"sentry_client=sentry.go/%s, sentry_key=%s", apiVersion, time.Now().Unix(), SDKVersion, dsn.publicKey)
 
 	if dsn.secretKey != "" {
 		auth = fmt.Sprintf("%s, sentry_secret=%s", auth, dsn.secretKey)

vendor/github.com/getsentry/sentry-go/integrations.go

@@ -2,6 +2,7 @@ package sentry
 
 import (
 	"fmt"
+	"os"
 	"regexp"
 	"runtime"
 	"runtime/debug"
@@ -26,7 +27,7 @@ func (mi *modulesIntegration) SetupOnce(client *Client) {
 	client.AddEventProcessor(mi.processor)
 }
 
-func (mi *modulesIntegration) processor(event *Event, hint *EventHint) *Event {
+func (mi *modulesIntegration) processor(event *Event, _ *EventHint) *Event {
 	if len(event.Modules) == 0 {
 		mi.once.Do(func() {
 			info, ok := debug.ReadBuildInfo()
@@ -69,7 +70,7 @@ func (ei *environmentIntegration) SetupOnce(client *Client) {
 	client.AddEventProcessor(ei.processor)
 }
 
-func (ei *environmentIntegration) processor(event *Event, hint *EventHint) *Event {
+func (ei *environmentIntegration) processor(event *Event, _ *EventHint) *Event {
 	// Initialize maps as necessary.
 	contextNames := []string{"device", "os", "runtime"}
 	if event.Contexts == nil {
@@ -134,7 +135,7 @@ func (iei *ignoreErrorsIntegration) SetupOnce(client *Client) {
 	client.AddEventProcessor(iei.processor)
 }
 
-func (iei *ignoreErrorsIntegration) processor(event *Event, hint *EventHint) *Event {
+func (iei *ignoreErrorsIntegration) processor(event *Event, _ *EventHint) *Event {
 	suspects := getIgnoreErrorsSuspects(event)
 
 	for _, suspect := range suspects {
@@ -177,6 +178,40 @@ func getIgnoreErrorsSuspects(event *Event) []string {
 	return suspects
 }
 
+// ================================
+// Ignore Transactions Integration
+// ================================
+
+type ignoreTransactionsIntegration struct {
+	ignoreTransactions []*regexp.Regexp
+}
+
+func (iei *ignoreTransactionsIntegration) Name() string {
+	return "IgnoreTransactions"
+}
+
+func (iei *ignoreTransactionsIntegration) SetupOnce(client *Client) {
+	iei.ignoreTransactions = transformStringsIntoRegexps(client.options.IgnoreTransactions)
+	client.AddEventProcessor(iei.processor)
+}
+
+func (iei *ignoreTransactionsIntegration) processor(event *Event, _ *EventHint) *Event {
+	suspect := event.Transaction
+	if suspect == "" {
+		return event
+	}
+
+	for _, pattern := range iei.ignoreTransactions {
+		if pattern.Match([]byte(suspect)) {
+			Logger.Printf("Transaction dropped due to being matched by `IgnoreTransactions` option."+
+				"| Value matched: %s | Filter used: %s", suspect, pattern)
+			return nil
+		}
+	}
+
+	return event
+}
+
 // ================================
 // Contextify Frames Integration
 // ================================
@@ -198,7 +233,7 @@ func (cfi *contextifyFramesIntegration) SetupOnce(client *Client) {
 	client.AddEventProcessor(cfi.processor)
 }
 
-func (cfi *contextifyFramesIntegration) processor(event *Event, hint *EventHint) *Event {
+func (cfi *contextifyFramesIntegration) processor(event *Event, _ *EventHint) *Event {
 	// Range over all exceptions
 	for _, ex := range event.Exception {
 		// If it has no stacktrace, just bail out
@@ -291,3 +326,66 @@ func (cfi *contextifyFramesIntegration) addContextLinesToFrame(frame Frame, line
 	}
 	return frame
 }
+
+// ================================
+// Global Tags Integration
+// ================================
+
+const envTagsPrefix = "SENTRY_TAGS_"
+
+type globalTagsIntegration struct {
+	tags    map[string]string
+	envTags map[string]string
+}
+
+func (ti *globalTagsIntegration) Name() string {
+	return "GlobalTags"
+}
+
+func (ti *globalTagsIntegration) SetupOnce(client *Client) {
+	ti.tags = make(map[string]string, len(client.options.Tags))
+	for k, v := range client.options.Tags {
+		ti.tags[k] = v
+	}
+
+	ti.envTags = loadEnvTags()
+
+	client.AddEventProcessor(ti.processor)
+}
+
+func (ti *globalTagsIntegration) processor(event *Event, _ *EventHint) *Event {
+	if len(ti.tags) == 0 && len(ti.envTags) == 0 {
+		return event
+	}
+
+	if event.Tags == nil {
+		event.Tags = make(map[string]string, len(ti.tags)+len(ti.envTags))
+	}
+
+	for k, v := range ti.tags {
+		if _, ok := event.Tags[k]; !ok {
+			event.Tags[k] = v
+		}
+	}
+
+	for k, v := range ti.envTags {
+		if _, ok := event.Tags[k]; !ok {
+			event.Tags[k] = v
+		}
+	}
+
+	return event
+}
+
+func loadEnvTags() map[string]string {
+	tags := map[string]string{}
+	for _, pair := range os.Environ() {
+		parts := strings.Split(pair, "=")
+		if !strings.HasPrefix(parts[0], envTagsPrefix) {
+			continue
+		}
+		tag := strings.TrimPrefix(parts[0], envTagsPrefix)
+		tags[tag] = parts[1]
+	}
+	return tags
+}

vendor/github.com/getsentry/sentry-go/interfaces.go

@@ -11,15 +11,14 @@ import (
 	"time"
 )
 
-// Protocol Docs (kinda)
+// eventType is the type of an error event.
-// https://github.com/getsentry/rust-sentry-types/blob/master/src/protocol/v7.rs
+const eventType = "event"
 
 // transactionType is the type of a transaction event.
 const transactionType = "transaction"
 
-// eventType is the type of an error event.
+// profileType is the type of a profile event.
-const eventType = "event"
+// currently, profiles are always sent as part of a transaction event.
-
 const profileType = "profile"
 
 // checkInType is the type of a check in event.
@@ -242,7 +241,7 @@ type Exception struct {
 	Type       string      `json:"type,omitempty"`  // used as the main issue title
 	Value      string      `json:"value,omitempty"` // used as the main issue subtitle
 	Module     string      `json:"module,omitempty"`
-	ThreadID   string      `json:"thread_id,omitempty"`
+	ThreadID   uint64      `json:"thread_id,omitempty"`
 	Stacktrace *Stacktrace `json:"stacktrace,omitempty"`
 	Mechanism  *Mechanism  `json:"mechanism,omitempty"`
 }
@@ -318,6 +317,7 @@ type Event struct {
 	Request     *Request               `json:"request,omitempty"`
 	Exception   []Exception            `json:"exception,omitempty"`
 	DebugMeta   *DebugMeta             `json:"debug_meta,omitempty"`
+	Attachments []*Attachment          `json:"-"`
 
 	// The fields below are only relevant for transactions.
 
@@ -334,7 +334,6 @@ type Event struct {
 	// The fields below are not part of the final JSON payload.
 
 	sdkMetaData SDKMetaData
-	attachments []*Attachment
 }
 
 // SetException appends the unwrapped errors to the event's exception list.

vendor/github.com/getsentry/sentry-go/profiler.go

@@ -236,18 +236,32 @@ func (p *profileRecorder) getBuckets(relativeStartNS, relativeEndNS uint64) (sam
 		return 0, nil, nil
 	}
 
-	// Search for the first item after the given startTime.
+	{ // Find the first item after the given startTime.
-	var start = end
+		var start = end
-	samplesCount = 0
+		var prevBucket *profileSamplesBucket
-	buckets = make([]*profileSamplesBucket, 0, int64((relativeEndNS-relativeStartNS)/uint64(profilerSamplingRate.Nanoseconds()))+1)
+		samplesCount = 0
-	for start.Value != nil {
+		buckets = make([]*profileSamplesBucket, 0, int64((relativeEndNS-relativeStartNS)/uint64(profilerSamplingRate.Nanoseconds()))+1)
-		var bucket = start.Value.(*profileSamplesBucket)
+		for start.Value != nil {
-		if bucket.relativeTimeNS < relativeStartNS {
+			var bucket = start.Value.(*profileSamplesBucket)
-			break
+
+			// If this bucket's time is before the requests start time, don't collect it (and stop iterating further).
+			if bucket.relativeTimeNS < relativeStartNS {
+				break
+			}
+
+			// If this bucket time is greater than previous the bucket's time, we have exhausted the whole ring buffer
+			// before we were able to find the start time. That means the start time is not present and we must break.
+			// This happens if the slice duration exceeds the ring buffer capacity.
+			if prevBucket != nil && bucket.relativeTimeNS > prevBucket.relativeTimeNS {
+				break
+			}
+
+			samplesCount += len(bucket.goIDs)
+			buckets = append(buckets, bucket)
+
+			start = start.Prev()
+			prevBucket = bucket
 		}
-		samplesCount += len(bucket.goIDs)
-		buckets = append(buckets, bucket)
-		start = start.Prev()
 	}
 
 	// Edge case - if the period requested was too short and we haven't collected enough samples.

vendor/github.com/getsentry/sentry-go/scope.go

@@ -344,7 +344,7 @@ func (scope *Scope) ApplyToEvent(event *Event, hint *EventHint) *Event {
 	}
 
 	if len(scope.attachments) > 0 {
-		event.attachments = append(event.attachments, scope.attachments...)
+		event.Attachments = append(event.Attachments, scope.attachments...)
 	}
 
 	if len(scope.tags) > 0 {

vendor/github.com/getsentry/sentry-go/sentry.go

@@ -6,15 +6,7 @@ import (
 )
 
 // The version of the SDK.
-// Deprecated: To be removed in 0.25.0. Use SDKVersion instead.
+const SDKVersion = "0.27.0"
-const Version = SDKVersion
-
-// The version of the SDK.
-const SDKVersion = "0.24.0"
-
-// The identifier of the SDK.
-// Deprecated: To be removed in 0.25.0. Use Client.GetSDKIdentifier() instead.
-const SDKIdentifier = "sentry.go"
 
 // apiVersion is the minimum version of the Sentry API compatible with the
 // sentry-go SDK.

vendor/github.com/getsentry/sentry-go/stacktrace.go

@@ -355,9 +355,7 @@ func callerFunctionName() string {
 // It replicates https://golang.org/pkg/debug/gosym/#Sym.PackageName, avoiding a
 // dependency on debug/gosym.
 func packageName(name string) string {
-	// A prefix of "type." and "go." is a compiler-generated symbol that doesn't belong to any package.
+	if isCompilerGeneratedSymbol(name) {
-	// See variable reservedimports in cmd/compile/internal/gc/subr.go
-	if strings.HasPrefix(name, "go.") || strings.HasPrefix(name, "type.") {
 		return ""
 	}
 

vendor/github.com/getsentry/sentry-go/stacktrace_below_go1.20.go

@@ -0,0 +1,15 @@
+//go:build !go1.20
+
+package sentry
+
+import "strings"
+
+func isCompilerGeneratedSymbol(name string) bool {
+	// In versions of Go below 1.20 a prefix of "type." and "go." is a
+	// compiler-generated symbol that doesn't belong to any package.
+	// See variable reservedimports in cmd/compile/internal/gc/subr.go
+	if strings.HasPrefix(name, "go.") || strings.HasPrefix(name, "type.") {
+		return true
+	}
+	return false
+}

vendor/github.com/getsentry/sentry-go/stacktrace_go1.20.go

@@ -0,0 +1,15 @@
+//go:build go1.20
+
+package sentry
+
+import "strings"
+
+func isCompilerGeneratedSymbol(name string) bool {
+	// In versions of Go 1.20 and above a prefix of "type:" and "go:" is a
+	// compiler-generated symbol that doesn't belong to any package.
+	// See variable reservedimports in cmd/compile/internal/gc/subr.go
+	if strings.HasPrefix(name, "go:") || strings.HasPrefix(name, "type:") {
+		return true
+	}
+	return false
+}

vendor/github.com/getsentry/sentry-go/tracing.go

@@ -49,11 +49,6 @@ type Span struct { //nolint: maligned // prefer readability over optimal memory
 	// parent refers to the immediate local parent span. A remote parent span is
 	// only referenced by setting ParentSpanID.
 	parent *Span
-	// isTransaction is true only for the root span of a local span tree. The
-	// root span is the first span started in a context. Note that a local root
-	// span may have a remote parent belonging to the same trace, therefore
-	// isTransaction depends on ctx and not on parent.
-	isTransaction bool
 	// recorder stores all spans in a transaction. Guaranteed to be non-nil.
 	recorder *spanRecorder
 	// span context, can only be set on transactions
@@ -114,9 +109,8 @@ func StartSpan(ctx context.Context, operation string, options ...SpanOption) *Sp
 		StartTime: time.Now(),
 		Sampled:   SampledUndefined,
 
-		ctx:           context.WithValue(ctx, spanContextKey{}, &span),
+		ctx:    context.WithValue(ctx, spanContextKey{}, &span),
-		parent:        parent,
+		parent: parent,
-		isTransaction: !hasParent,
 	}
 
 	if hasParent {
@@ -257,7 +251,7 @@ func (s *Span) SetContext(key string, value Context) {
 
 // IsTransaction checks if the given span is a transaction.
 func (s *Span) IsTransaction() bool {
-	return s.isTransaction
+	return s.parent == nil
 }
 
 // GetTransaction returns the transaction that contains this span.
@@ -289,7 +283,7 @@ func (s *Span) GetTransaction() *Span {
 // func (s *Span) TransactionName() string
 // func (s *Span) SetTransactionName(name string)
 
-// ToSentryTrace returns the seralized TraceParentContext from a transaction/sapn.
+// ToSentryTrace returns the seralized TraceParentContext from a transaction/span.
 // Use this function to propagate the TraceParentContext to a downstream SDK,
 // either as the value of the "sentry-trace" HTTP header, or as an html "sentry-trace" meta tag.
 func (s *Span) ToSentryTrace() string {
@@ -326,7 +320,7 @@ func (s *Span) ToBaggage() string {
 // SetDynamicSamplingContext sets the given dynamic sampling context on the
 // current transaction.
 func (s *Span) SetDynamicSamplingContext(dsc DynamicSamplingContext) {
-	if s.isTransaction {
+	if s.IsTransaction() {
 		s.dynamicSamplingContext = dsc
 	}
 }
@@ -391,7 +385,7 @@ func (s *Span) updateFromSentryTrace(header []byte) (updated bool) {
 }
 
 func (s *Span) updateFromBaggage(header []byte) {
-	if s.isTransaction {
+	if s.IsTransaction() {
 		dsc, err := DynamicSamplingContextFromHeader(header)
 		if err != nil {
 			return
@@ -452,7 +446,7 @@ func (s *Span) sample() Sampled {
 	// Note: non-transaction should always have a parent, but we check both
 	// conditions anyway -- the first for semantic meaning, the second to
 	// avoid a nil pointer dereference.
-	if !s.isTransaction && s.parent != nil {
+	if !s.IsTransaction() && s.parent != nil {
 		return s.parent.Sampled
 	}
 
@@ -516,7 +510,7 @@ func (s *Span) toEvent() *Event {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	if !s.isTransaction {
+	if !s.IsTransaction() {
 		return nil // only transactions can be transformed into events
 	}
 
@@ -829,17 +823,6 @@ func (s Sampled) Bool() bool {
 // A SpanOption is a function that can modify the properties of a span.
 type SpanOption func(s *Span)
 
-// The TransactionName option sets the name of the current transaction.
-//
-// A span tree has a single transaction name, therefore using this option when
-// starting a span affects the span tree as a whole, potentially overwriting a
-// name set previously.
-//
-// Deprecated: Use WithTransactionName() instead.
-func TransactionName(name string) SpanOption {
-	return WithTransactionName(name)
-}
-
 // WithTransactionName option sets the name of the current transaction.
 //
 // A span tree has a single transaction name, therefore using this option when
@@ -851,11 +834,11 @@ func WithTransactionName(name string) SpanOption {
 	}
 }
 
-// OpName sets the operation name for a given span.
+// WithDescription sets the description of a span.
-//
+func WithDescription(description string) SpanOption {
-// Deprecated: Use WithOpName() instead.
+	return func(s *Span) {
-func OpName(name string) SpanOption {
+		s.Description = description
-	return WithOpName(name)
+	}
 }
 
 // WithOpName sets the operation name for a given span.
@@ -865,13 +848,6 @@ func WithOpName(name string) SpanOption {
 	}
 }
 
-// TransctionSource sets the source of the transaction name.
-//
-// Deprecated: Use WithTransactionSource() instead.
-func TransctionSource(source TransactionSource) SpanOption {
-	return WithTransactionSource(source)
-}
-
 // WithTransactionSource sets the source of the transaction name.
 //
 // Note: if the transaction source is not a valid source (as described
@@ -883,13 +859,6 @@ func WithTransactionSource(source TransactionSource) SpanOption {
 	}
 }
 
-// SpanSampled updates the sampling flag for a given span.
-//
-// Deprecated: Use WithSpanSampled() instead.
-func SpanSampled(sampled Sampled) SpanOption {
-	return WithSpanSampled(sampled)
-}
-
 // WithSpanSampled updates the sampling flag for a given span.
 func WithSpanSampled(sampled Sampled) SpanOption {
 	return func(s *Span) {

vendor/github.com/getsentry/sentry-go/transport.go

@@ -185,7 +185,7 @@ func envelopeFromBody(event *Event, dsn *Dsn, sentAt time.Time, body json.RawMes
 	}
 
 	// Attachments
-	for _, attachment := range event.attachments {
+	for _, attachment := range event.Attachments {
 		if err := encodeAttachment(enc, &b, attachment); err != nil {
 			return nil, err
 		}

vendor/github.com/getsentry/sentry-go/util.go

@@ -78,24 +78,28 @@ func defaultRelease() (release string) {
 	// 	v1.0.1-0-g9de4
 	// 	v2.0-8-g77df-dirty
 	// 	4f72d7
-	cmd := exec.Command("git", "describe", "--long", "--always", "--dirty")
+	if _, err := exec.LookPath("git"); err == nil {
-	b, err := cmd.Output()
+		cmd := exec.Command("git", "describe", "--long", "--always", "--dirty")
-	if err != nil {
+		b, err := cmd.Output()
-		// Either Git is not available or the current directory is not a
+		if err != nil {
-		// Git repository.
+			// Either Git is not available or the current directory is not a
-		var s strings.Builder
+			// Git repository.
-		fmt.Fprintf(&s, "Release detection failed: %v", err)
+			var s strings.Builder
-		if err, ok := err.(*exec.ExitError); ok && len(err.Stderr) > 0 {
+			fmt.Fprintf(&s, "Release detection failed: %v", err)
-			fmt.Fprintf(&s, ": %s", err.Stderr)
+			if err, ok := err.(*exec.ExitError); ok && len(err.Stderr) > 0 {
+				fmt.Fprintf(&s, ": %s", err.Stderr)
+			}
+			Logger.Print(s.String())
+		} else {
+			release = strings.TrimSpace(string(b))
+			Logger.Printf("Using release from Git: %s", release)
+			return release
 		}
-		Logger.Print(s.String())
-		Logger.Print("Some Sentry features will not be available. See https://docs.sentry.io/product/releases/.")
-		Logger.Print("To stop seeing this message, pass a Release to sentry.Init or set the SENTRY_RELEASE environment variable.")
-		return ""
 	}
-	release = strings.TrimSpace(string(b))
+
-	Logger.Printf("Using release from Git: %s", release)
+	Logger.Print("Some Sentry features will not be available. See https://docs.sentry.io/product/releases/.")
-	return release
+	Logger.Print("To stop seeing this message, pass a Release to sentry.Init or set the SENTRY_RELEASE environment variable.")
+	return ""
 }
 
 func revisionFromBuildInfo(info *debug.BuildInfo) string {

vendor/github.com/golang/protobuf/AUTHORS

@@ -1,3 +0,0 @@
-# This source code refers to The Go Authors for copyright purposes.
-# The master list of authors is in the main Go distribution,
-# visible at http://tip.golang.org/AUTHORS.

vendor/github.com/golang/protobuf/CONTRIBUTORS

@@ -1,3 +0,0 @@
-# This source code was written by the Go contributors.
-# The master list of contributors is in the main Go distribution,
-# visible at http://tip.golang.org/CONTRIBUTORS.

vendor/github.com/golang/protobuf/LICENSE

@@ -1,28 +0,0 @@
-Copyright 2010 The Go Authors.  All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-    * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-    * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-

vendor/github.com/golang/protobuf/proto/buffer.go

@@ -1,324 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package proto
-
-import (
-	"errors"
-	"fmt"
-
-	"google.golang.org/protobuf/encoding/prototext"
-	"google.golang.org/protobuf/encoding/protowire"
-	"google.golang.org/protobuf/runtime/protoimpl"
-)
-
-const (
-	WireVarint     = 0
-	WireFixed32    = 5
-	WireFixed64    = 1
-	WireBytes      = 2
-	WireStartGroup = 3
-	WireEndGroup   = 4
-)
-
-// EncodeVarint returns the varint encoded bytes of v.
-func EncodeVarint(v uint64) []byte {
-	return protowire.AppendVarint(nil, v)
-}
-
-// SizeVarint returns the length of the varint encoded bytes of v.
-// This is equal to len(EncodeVarint(v)).
-func SizeVarint(v uint64) int {
-	return protowire.SizeVarint(v)
-}
-
-// DecodeVarint parses a varint encoded integer from b,
-// returning the integer value and the length of the varint.
-// It returns (0, 0) if there is a parse error.
-func DecodeVarint(b []byte) (uint64, int) {
-	v, n := protowire.ConsumeVarint(b)
-	if n < 0 {
-		return 0, 0
-	}
-	return v, n
-}
-
-// Buffer is a buffer for encoding and decoding the protobuf wire format.
-// It may be reused between invocations to reduce memory usage.
-type Buffer struct {
-	buf           []byte
-	idx           int
-	deterministic bool
-}
-
-// NewBuffer allocates a new Buffer initialized with buf,
-// where the contents of buf are considered the unread portion of the buffer.
-func NewBuffer(buf []byte) *Buffer {
-	return &Buffer{buf: buf}
-}
-
-// SetDeterministic specifies whether to use deterministic serialization.
-//
-// Deterministic serialization guarantees that for a given binary, equal
-// messages will always be serialized to the same bytes. This implies:
-//
-//   - Repeated serialization of a message will return the same bytes.
-//   - Different processes of the same binary (which may be executing on
-//     different machines) will serialize equal messages to the same bytes.
-//
-// Note that the deterministic serialization is NOT canonical across
-// languages. It is not guaranteed to remain stable over time. It is unstable
-// across different builds with schema changes due to unknown fields.
-// Users who need canonical serialization (e.g., persistent storage in a
-// canonical form, fingerprinting, etc.) should define their own
-// canonicalization specification and implement their own serializer rather
-// than relying on this API.
-//
-// If deterministic serialization is requested, map entries will be sorted
-// by keys in lexographical order. This is an implementation detail and
-// subject to change.
-func (b *Buffer) SetDeterministic(deterministic bool) {
-	b.deterministic = deterministic
-}
-
-// SetBuf sets buf as the internal buffer,
-// where the contents of buf are considered the unread portion of the buffer.
-func (b *Buffer) SetBuf(buf []byte) {
-	b.buf = buf
-	b.idx = 0
-}
-
-// Reset clears the internal buffer of all written and unread data.
-func (b *Buffer) Reset() {
-	b.buf = b.buf[:0]
-	b.idx = 0
-}
-
-// Bytes returns the internal buffer.
-func (b *Buffer) Bytes() []byte {
-	return b.buf
-}
-
-// Unread returns the unread portion of the buffer.
-func (b *Buffer) Unread() []byte {
-	return b.buf[b.idx:]
-}
-
-// Marshal appends the wire-format encoding of m to the buffer.
-func (b *Buffer) Marshal(m Message) error {
-	var err error
-	b.buf, err = marshalAppend(b.buf, m, b.deterministic)
-	return err
-}
-
-// Unmarshal parses the wire-format message in the buffer and
-// places the decoded results in m.
-// It does not reset m before unmarshaling.
-func (b *Buffer) Unmarshal(m Message) error {
-	err := UnmarshalMerge(b.Unread(), m)
-	b.idx = len(b.buf)
-	return err
-}
-
-type unknownFields struct{ XXX_unrecognized protoimpl.UnknownFields }
-
-func (m *unknownFields) String() string { panic("not implemented") }
-func (m *unknownFields) Reset()         { panic("not implemented") }
-func (m *unknownFields) ProtoMessage()  { panic("not implemented") }
-
-// DebugPrint dumps the encoded bytes of b with a header and footer including s
-// to stdout. This is only intended for debugging.
-func (*Buffer) DebugPrint(s string, b []byte) {
-	m := MessageReflect(new(unknownFields))
-	m.SetUnknown(b)
-	b, _ = prototext.MarshalOptions{AllowPartial: true, Indent: "\t"}.Marshal(m.Interface())
-	fmt.Printf("==== %s ====\n%s==== %s ====\n", s, b, s)
-}
-
-// EncodeVarint appends an unsigned varint encoding to the buffer.
-func (b *Buffer) EncodeVarint(v uint64) error {
-	b.buf = protowire.AppendVarint(b.buf, v)
-	return nil
-}
-
-// EncodeZigzag32 appends a 32-bit zig-zag varint encoding to the buffer.
-func (b *Buffer) EncodeZigzag32(v uint64) error {
-	return b.EncodeVarint(uint64((uint32(v) << 1) ^ uint32((int32(v) >> 31))))
-}
-
-// EncodeZigzag64 appends a 64-bit zig-zag varint encoding to the buffer.
-func (b *Buffer) EncodeZigzag64(v uint64) error {
-	return b.EncodeVarint(uint64((uint64(v) << 1) ^ uint64((int64(v) >> 63))))
-}
-
-// EncodeFixed32 appends a 32-bit little-endian integer to the buffer.
-func (b *Buffer) EncodeFixed32(v uint64) error {
-	b.buf = protowire.AppendFixed32(b.buf, uint32(v))
-	return nil
-}
-
-// EncodeFixed64 appends a 64-bit little-endian integer to the buffer.
-func (b *Buffer) EncodeFixed64(v uint64) error {
-	b.buf = protowire.AppendFixed64(b.buf, uint64(v))
-	return nil
-}
-
-// EncodeRawBytes appends a length-prefixed raw bytes to the buffer.
-func (b *Buffer) EncodeRawBytes(v []byte) error {
-	b.buf = protowire.AppendBytes(b.buf, v)
-	return nil
-}
-
-// EncodeStringBytes appends a length-prefixed raw bytes to the buffer.
-// It does not validate whether v contains valid UTF-8.
-func (b *Buffer) EncodeStringBytes(v string) error {
-	b.buf = protowire.AppendString(b.buf, v)
-	return nil
-}
-
-// EncodeMessage appends a length-prefixed encoded message to the buffer.
-func (b *Buffer) EncodeMessage(m Message) error {
-	var err error
-	b.buf = protowire.AppendVarint(b.buf, uint64(Size(m)))
-	b.buf, err = marshalAppend(b.buf, m, b.deterministic)
-	return err
-}
-
-// DecodeVarint consumes an encoded unsigned varint from the buffer.
-func (b *Buffer) DecodeVarint() (uint64, error) {
-	v, n := protowire.ConsumeVarint(b.buf[b.idx:])
-	if n < 0 {
-		return 0, protowire.ParseError(n)
-	}
-	b.idx += n
-	return uint64(v), nil
-}
-
-// DecodeZigzag32 consumes an encoded 32-bit zig-zag varint from the buffer.
-func (b *Buffer) DecodeZigzag32() (uint64, error) {
-	v, err := b.DecodeVarint()
-	if err != nil {
-		return 0, err
-	}
-	return uint64((uint32(v) >> 1) ^ uint32((int32(v&1)<<31)>>31)), nil
-}
-
-// DecodeZigzag64 consumes an encoded 64-bit zig-zag varint from the buffer.
-func (b *Buffer) DecodeZigzag64() (uint64, error) {
-	v, err := b.DecodeVarint()
-	if err != nil {
-		return 0, err
-	}
-	return uint64((uint64(v) >> 1) ^ uint64((int64(v&1)<<63)>>63)), nil
-}
-
-// DecodeFixed32 consumes a 32-bit little-endian integer from the buffer.
-func (b *Buffer) DecodeFixed32() (uint64, error) {
-	v, n := protowire.ConsumeFixed32(b.buf[b.idx:])
-	if n < 0 {
-		return 0, protowire.ParseError(n)
-	}
-	b.idx += n
-	return uint64(v), nil
-}
-
-// DecodeFixed64 consumes a 64-bit little-endian integer from the buffer.
-func (b *Buffer) DecodeFixed64() (uint64, error) {
-	v, n := protowire.ConsumeFixed64(b.buf[b.idx:])
-	if n < 0 {
-		return 0, protowire.ParseError(n)
-	}
-	b.idx += n
-	return uint64(v), nil
-}
-
-// DecodeRawBytes consumes a length-prefixed raw bytes from the buffer.
-// If alloc is specified, it returns a copy the raw bytes
-// rather than a sub-slice of the buffer.
-func (b *Buffer) DecodeRawBytes(alloc bool) ([]byte, error) {
-	v, n := protowire.ConsumeBytes(b.buf[b.idx:])
-	if n < 0 {
-		return nil, protowire.ParseError(n)
-	}
-	b.idx += n
-	if alloc {
-		v = append([]byte(nil), v...)
-	}
-	return v, nil
-}
-
-// DecodeStringBytes consumes a length-prefixed raw bytes from the buffer.
-// It does not validate whether the raw bytes contain valid UTF-8.
-func (b *Buffer) DecodeStringBytes() (string, error) {
-	v, n := protowire.ConsumeString(b.buf[b.idx:])
-	if n < 0 {
-		return "", protowire.ParseError(n)
-	}
-	b.idx += n
-	return v, nil
-}
-
-// DecodeMessage consumes a length-prefixed message from the buffer.
-// It does not reset m before unmarshaling.
-func (b *Buffer) DecodeMessage(m Message) error {
-	v, err := b.DecodeRawBytes(false)
-	if err != nil {
-		return err
-	}
-	return UnmarshalMerge(v, m)
-}
-
-// DecodeGroup consumes a message group from the buffer.
-// It assumes that the start group marker has already been consumed and
-// consumes all bytes until (and including the end group marker).
-// It does not reset m before unmarshaling.
-func (b *Buffer) DecodeGroup(m Message) error {
-	v, n, err := consumeGroup(b.buf[b.idx:])
-	if err != nil {
-		return err
-	}
-	b.idx += n
-	return UnmarshalMerge(v, m)
-}
-
-// consumeGroup parses b until it finds an end group marker, returning
-// the raw bytes of the message (excluding the end group marker) and the
-// the total length of the message (including the end group marker).
-func consumeGroup(b []byte) ([]byte, int, error) {
-	b0 := b
-	depth := 1 // assume this follows a start group marker
-	for {
-		_, wtyp, tagLen := protowire.ConsumeTag(b)
-		if tagLen < 0 {
-			return nil, 0, protowire.ParseError(tagLen)
-		}
-		b = b[tagLen:]
-
-		var valLen int
-		switch wtyp {
-		case protowire.VarintType:
-			_, valLen = protowire.ConsumeVarint(b)
-		case protowire.Fixed32Type:
-			_, valLen = protowire.ConsumeFixed32(b)
-		case protowire.Fixed64Type:
-			_, valLen = protowire.ConsumeFixed64(b)
-		case protowire.BytesType:
-			_, valLen = protowire.ConsumeBytes(b)
-		case protowire.StartGroupType:
-			depth++
-		case protowire.EndGroupType:
-			depth--
-		default:
-			return nil, 0, errors.New("proto: cannot parse reserved wire type")
-		}
-		if valLen < 0 {
-			return nil, 0, protowire.ParseError(valLen)
-		}
-		b = b[valLen:]
-
-		if depth == 0 {
-			return b0[:len(b0)-len(b)-tagLen], len(b0) - len(b), nil
-		}
-	}
-}

vendor/github.com/golang/protobuf/proto/defaults.go

@@ -1,63 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package proto
-
-import (
-	"google.golang.org/protobuf/reflect/protoreflect"
-)
-
-// SetDefaults sets unpopulated scalar fields to their default values.
-// Fields within a oneof are not set even if they have a default value.
-// SetDefaults is recursively called upon any populated message fields.
-func SetDefaults(m Message) {
-	if m != nil {
-		setDefaults(MessageReflect(m))
-	}
-}
-
-func setDefaults(m protoreflect.Message) {
-	fds := m.Descriptor().Fields()
-	for i := 0; i < fds.Len(); i++ {
-		fd := fds.Get(i)
-		if !m.Has(fd) {
-			if fd.HasDefault() && fd.ContainingOneof() == nil {
-				v := fd.Default()
-				if fd.Kind() == protoreflect.BytesKind {
-					v = protoreflect.ValueOf(append([]byte(nil), v.Bytes()...)) // copy the default bytes
-				}
-				m.Set(fd, v)
-			}
-			continue
-		}
-	}
-
-	m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
-		switch {
-		// Handle singular message.
-		case fd.Cardinality() != protoreflect.Repeated:
-			if fd.Message() != nil {
-				setDefaults(m.Get(fd).Message())
-			}
-		// Handle list of messages.
-		case fd.IsList():
-			if fd.Message() != nil {
-				ls := m.Get(fd).List()
-				for i := 0; i < ls.Len(); i++ {
-					setDefaults(ls.Get(i).Message())
-				}
-			}
-		// Handle map of messages.
-		case fd.IsMap():
-			if fd.MapValue().Message() != nil {
-				ms := m.Get(fd).Map()
-				ms.Range(func(_ protoreflect.MapKey, v protoreflect.Value) bool {
-					setDefaults(v.Message())
-					return true
-				})
-			}
-		}
-		return true
-	})
-}

vendor/github.com/golang/protobuf/proto/deprecated.go

@@ -1,113 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package proto
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"strconv"
-
-	protoV2 "google.golang.org/protobuf/proto"
-)
-
-var (
-	// Deprecated: No longer returned.
-	ErrNil = errors.New("proto: Marshal called with nil")
-
-	// Deprecated: No longer returned.
-	ErrTooLarge = errors.New("proto: message encodes to over 2 GB")
-
-	// Deprecated: No longer returned.
-	ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for oneof")
-)
-
-// Deprecated: Do not use.
-type Stats struct{ Emalloc, Dmalloc, Encode, Decode, Chit, Cmiss, Size uint64 }
-
-// Deprecated: Do not use.
-func GetStats() Stats { return Stats{} }
-
-// Deprecated: Do not use.
-func MarshalMessageSet(interface{}) ([]byte, error) {
-	return nil, errors.New("proto: not implemented")
-}
-
-// Deprecated: Do not use.
-func UnmarshalMessageSet([]byte, interface{}) error {
-	return errors.New("proto: not implemented")
-}
-
-// Deprecated: Do not use.
-func MarshalMessageSetJSON(interface{}) ([]byte, error) {
-	return nil, errors.New("proto: not implemented")
-}
-
-// Deprecated: Do not use.
-func UnmarshalMessageSetJSON([]byte, interface{}) error {
-	return errors.New("proto: not implemented")
-}
-
-// Deprecated: Do not use.
-func RegisterMessageSetType(Message, int32, string) {}
-
-// Deprecated: Do not use.
-func EnumName(m map[int32]string, v int32) string {
-	s, ok := m[v]
-	if ok {
-		return s
-	}
-	return strconv.Itoa(int(v))
-}
-
-// Deprecated: Do not use.
-func UnmarshalJSONEnum(m map[string]int32, data []byte, enumName string) (int32, error) {
-	if data[0] == '"' {
-		// New style: enums are strings.
-		var repr string
-		if err := json.Unmarshal(data, &repr); err != nil {
-			return -1, err
-		}
-		val, ok := m[repr]
-		if !ok {
-			return 0, fmt.Errorf("unrecognized enum %s value %q", enumName, repr)
-		}
-		return val, nil
-	}
-	// Old style: enums are ints.
-	var val int32
-	if err := json.Unmarshal(data, &val); err != nil {
-		return 0, fmt.Errorf("cannot unmarshal %#q into enum %s", data, enumName)
-	}
-	return val, nil
-}
-
-// Deprecated: Do not use; this type existed for intenal-use only.
-type InternalMessageInfo struct{}
-
-// Deprecated: Do not use; this method existed for intenal-use only.
-func (*InternalMessageInfo) DiscardUnknown(m Message) {
-	DiscardUnknown(m)
-}
-
-// Deprecated: Do not use; this method existed for intenal-use only.
-func (*InternalMessageInfo) Marshal(b []byte, m Message, deterministic bool) ([]byte, error) {
-	return protoV2.MarshalOptions{Deterministic: deterministic}.MarshalAppend(b, MessageV2(m))
-}
-
-// Deprecated: Do not use; this method existed for intenal-use only.
-func (*InternalMessageInfo) Merge(dst, src Message) {
-	protoV2.Merge(MessageV2(dst), MessageV2(src))
-}
-
-// Deprecated: Do not use; this method existed for intenal-use only.
-func (*InternalMessageInfo) Size(m Message) int {
-	return protoV2.Size(MessageV2(m))
-}
-
-// Deprecated: Do not use; this method existed for intenal-use only.
-func (*InternalMessageInfo) Unmarshal(m Message, b []byte) error {
-	return protoV2.UnmarshalOptions{Merge: true}.Unmarshal(b, MessageV2(m))
-}

vendor/github.com/golang/protobuf/proto/discard.go

@@ -1,58 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package proto
-
-import (
-	"google.golang.org/protobuf/reflect/protoreflect"
-)
-
-// DiscardUnknown recursively discards all unknown fields from this message
-// and all embedded messages.
-//
-// When unmarshaling a message with unrecognized fields, the tags and values
-// of such fields are preserved in the Message. This allows a later call to
-// marshal to be able to produce a message that continues to have those
-// unrecognized fields. To avoid this, DiscardUnknown is used to
-// explicitly clear the unknown fields after unmarshaling.
-func DiscardUnknown(m Message) {
-	if m != nil {
-		discardUnknown(MessageReflect(m))
-	}
-}
-
-func discardUnknown(m protoreflect.Message) {
-	m.Range(func(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool {
-		switch {
-		// Handle singular message.
-		case fd.Cardinality() != protoreflect.Repeated:
-			if fd.Message() != nil {
-				discardUnknown(m.Get(fd).Message())
-			}
-		// Handle list of messages.
-		case fd.IsList():
-			if fd.Message() != nil {
-				ls := m.Get(fd).List()
-				for i := 0; i < ls.Len(); i++ {
-					discardUnknown(ls.Get(i).Message())
-				}
-			}
-		// Handle map of messages.
-		case fd.IsMap():
-			if fd.MapValue().Message() != nil {
-				ms := m.Get(fd).Map()
-				ms.Range(func(_ protoreflect.MapKey, v protoreflect.Value) bool {
-					discardUnknown(v.Message())
-					return true
-				})
-			}
-		}
-		return true
-	})
-
-	// Discard unknown fields.
-	if len(m.GetUnknown()) > 0 {
-		m.SetUnknown(nil)
-	}
-}

vendor/github.com/golang/protobuf/proto/extensions.go

@@ -1,356 +0,0 @@
-// Copyright 2010 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package proto
-
-import (
-	"errors"
-	"fmt"
-	"reflect"
-
-	"google.golang.org/protobuf/encoding/protowire"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/reflect/protoreflect"
-	"google.golang.org/protobuf/reflect/protoregistry"
-	"google.golang.org/protobuf/runtime/protoiface"
-	"google.golang.org/protobuf/runtime/protoimpl"
-)
-
-type (
-	// ExtensionDesc represents an extension descriptor and
-	// is used to interact with an extension field in a message.
-	//
-	// Variables of this type are generated in code by protoc-gen-go.
-	ExtensionDesc = protoimpl.ExtensionInfo
-
-	// ExtensionRange represents a range of message extensions.
-	// Used in code generated by protoc-gen-go.
-	ExtensionRange = protoiface.ExtensionRangeV1
-
-	// Deprecated: Do not use; this is an internal type.
-	Extension = protoimpl.ExtensionFieldV1
-
-	// Deprecated: Do not use; this is an internal type.
-	XXX_InternalExtensions = protoimpl.ExtensionFields
-)
-
-// ErrMissingExtension reports whether the extension was not present.
-var ErrMissingExtension = errors.New("proto: missing extension")
-
-var errNotExtendable = errors.New("proto: not an extendable proto.Message")
-
-// HasExtension reports whether the extension field is present in m
-// either as an explicitly populated field or as an unknown field.
-func HasExtension(m Message, xt *ExtensionDesc) (has bool) {
-	mr := MessageReflect(m)
-	if mr == nil || !mr.IsValid() {
-		return false
-	}
-
-	// Check whether any populated known field matches the field number.
-	xtd := xt.TypeDescriptor()
-	if isValidExtension(mr.Descriptor(), xtd) {
-		has = mr.Has(xtd)
-	} else {
-		mr.Range(func(fd protoreflect.FieldDescriptor, _ protoreflect.Value) bool {
-			has = int32(fd.Number()) == xt.Field
-			return !has
-		})
-	}
-
-	// Check whether any unknown field matches the field number.
-	for b := mr.GetUnknown(); !has && len(b) > 0; {
-		num, _, n := protowire.ConsumeField(b)
-		has = int32(num) == xt.Field
-		b = b[n:]
-	}
-	return has
-}
-
-// ClearExtension removes the extension field from m
-// either as an explicitly populated field or as an unknown field.
-func ClearExtension(m Message, xt *ExtensionDesc) {
-	mr := MessageReflect(m)
-	if mr == nil || !mr.IsValid() {
-		return
-	}
-
-	xtd := xt.TypeDescriptor()
-	if isValidExtension(mr.Descriptor(), xtd) {
-		mr.Clear(xtd)
-	} else {
-		mr.Range(func(fd protoreflect.FieldDescriptor, _ protoreflect.Value) bool {
-			if int32(fd.Number()) == xt.Field {
-				mr.Clear(fd)
-				return false
-			}
-			return true
-		})
-	}
-	clearUnknown(mr, fieldNum(xt.Field))
-}
-
-// ClearAllExtensions clears all extensions from m.
-// This includes populated fields and unknown fields in the extension range.
-func ClearAllExtensions(m Message) {
-	mr := MessageReflect(m)
-	if mr == nil || !mr.IsValid() {
-		return
-	}
-
-	mr.Range(func(fd protoreflect.FieldDescriptor, _ protoreflect.Value) bool {
-		if fd.IsExtension() {
-			mr.Clear(fd)
-		}
-		return true
-	})
-	clearUnknown(mr, mr.Descriptor().ExtensionRanges())
-}
-
-// GetExtension retrieves a proto2 extended field from m.
-//
-// If the descriptor is type complete (i.e., ExtensionDesc.ExtensionType is non-nil),
-// then GetExtension parses the encoded field and returns a Go value of the specified type.
-// If the field is not present, then the default value is returned (if one is specified),
-// otherwise ErrMissingExtension is reported.
-//
-// If the descriptor is type incomplete (i.e., ExtensionDesc.ExtensionType is nil),
-// then GetExtension returns the raw encoded bytes for the extension field.
-func GetExtension(m Message, xt *ExtensionDesc) (interface{}, error) {
-	mr := MessageReflect(m)
-	if mr == nil || !mr.IsValid() || mr.Descriptor().ExtensionRanges().Len() == 0 {
-		return nil, errNotExtendable
-	}
-
-	// Retrieve the unknown fields for this extension field.
-	var bo protoreflect.RawFields
-	for bi := mr.GetUnknown(); len(bi) > 0; {
-		num, _, n := protowire.ConsumeField(bi)
-		if int32(num) == xt.Field {
-			bo = append(bo, bi[:n]...)
-		}
-		bi = bi[n:]
-	}
-
-	// For type incomplete descriptors, only retrieve the unknown fields.
-	if xt.ExtensionType == nil {
-		return []byte(bo), nil
-	}
-
-	// If the extension field only exists as unknown fields, unmarshal it.
-	// This is rarely done since proto.Unmarshal eagerly unmarshals extensions.
-	xtd := xt.TypeDescriptor()
-	if !isValidExtension(mr.Descriptor(), xtd) {
-		return nil, fmt.Errorf("proto: bad extended type; %T does not extend %T", xt.ExtendedType, m)
-	}
-	if !mr.Has(xtd) && len(bo) > 0 {
-		m2 := mr.New()
-		if err := (proto.UnmarshalOptions{
-			Resolver: extensionResolver{xt},
-		}.Unmarshal(bo, m2.Interface())); err != nil {
-			return nil, err
-		}
-		if m2.Has(xtd) {
-			mr.Set(xtd, m2.Get(xtd))
-			clearUnknown(mr, fieldNum(xt.Field))
-		}
-	}
-
-	// Check whether the message has the extension field set or a default.
-	var pv protoreflect.Value
-	switch {
-	case mr.Has(xtd):
-		pv = mr.Get(xtd)
-	case xtd.HasDefault():
-		pv = xtd.Default()
-	default:
-		return nil, ErrMissingExtension
-	}
-
-	v := xt.InterfaceOf(pv)
-	rv := reflect.ValueOf(v)
-	if isScalarKind(rv.Kind()) {
-		rv2 := reflect.New(rv.Type())
-		rv2.Elem().Set(rv)
-		v = rv2.Interface()
-	}
-	return v, nil
-}
-
-// extensionResolver is a custom extension resolver that stores a single
-// extension type that takes precedence over the global registry.
-type extensionResolver struct{ xt protoreflect.ExtensionType }
-
-func (r extensionResolver) FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error) {
-	if xtd := r.xt.TypeDescriptor(); xtd.FullName() == field {
-		return r.xt, nil
-	}
-	return protoregistry.GlobalTypes.FindExtensionByName(field)
-}
-
-func (r extensionResolver) FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error) {
-	if xtd := r.xt.TypeDescriptor(); xtd.ContainingMessage().FullName() == message && xtd.Number() == field {
-		return r.xt, nil
-	}
-	return protoregistry.GlobalTypes.FindExtensionByNumber(message, field)
-}
-
-// GetExtensions returns a list of the extensions values present in m,
-// corresponding with the provided list of extension descriptors, xts.
-// If an extension is missing in m, the corresponding value is nil.
-func GetExtensions(m Message, xts []*ExtensionDesc) ([]interface{}, error) {
-	mr := MessageReflect(m)
-	if mr == nil || !mr.IsValid() {
-		return nil, errNotExtendable
-	}
-
-	vs := make([]interface{}, len(xts))
-	for i, xt := range xts {
-		v, err := GetExtension(m, xt)
-		if err != nil {
-			if err == ErrMissingExtension {
-				continue
-			}
-			return vs, err
-		}
-		vs[i] = v
-	}
-	return vs, nil
-}
-
-// SetExtension sets an extension field in m to the provided value.
-func SetExtension(m Message, xt *ExtensionDesc, v interface{}) error {
-	mr := MessageReflect(m)
-	if mr == nil || !mr.IsValid() || mr.Descriptor().ExtensionRanges().Len() == 0 {
-		return errNotExtendable
-	}
-
-	rv := reflect.ValueOf(v)
-	if reflect.TypeOf(v) != reflect.TypeOf(xt.ExtensionType) {
-		return fmt.Errorf("proto: bad extension value type. got: %T, want: %T", v, xt.ExtensionType)
-	}
-	if rv.Kind() == reflect.Ptr {
-		if rv.IsNil() {
-			return fmt.Errorf("proto: SetExtension called with nil value of type %T", v)
-		}
-		if isScalarKind(rv.Elem().Kind()) {
-			v = rv.Elem().Interface()
-		}
-	}
-
-	xtd := xt.TypeDescriptor()
-	if !isValidExtension(mr.Descriptor(), xtd) {
-		return fmt.Errorf("proto: bad extended type; %T does not extend %T", xt.ExtendedType, m)
-	}
-	mr.Set(xtd, xt.ValueOf(v))
-	clearUnknown(mr, fieldNum(xt.Field))
-	return nil
-}
-
-// SetRawExtension inserts b into the unknown fields of m.
-//
-// Deprecated: Use Message.ProtoReflect.SetUnknown instead.
-func SetRawExtension(m Message, fnum int32, b []byte) {
-	mr := MessageReflect(m)
-	if mr == nil || !mr.IsValid() {
-		return
-	}
-
-	// Verify that the raw field is valid.
-	for b0 := b; len(b0) > 0; {
-		num, _, n := protowire.ConsumeField(b0)
-		if int32(num) != fnum {
-			panic(fmt.Sprintf("mismatching field number: got %d, want %d", num, fnum))
-		}
-		b0 = b0[n:]
-	}
-
-	ClearExtension(m, &ExtensionDesc{Field: fnum})
-	mr.SetUnknown(append(mr.GetUnknown(), b...))
-}
-
-// ExtensionDescs returns a list of extension descriptors found in m,
-// containing descriptors for both populated extension fields in m and
-// also unknown fields of m that are in the extension range.
-// For the later case, an type incomplete descriptor is provided where only
-// the ExtensionDesc.Field field is populated.
-// The order of the extension descriptors is undefined.
-func ExtensionDescs(m Message) ([]*ExtensionDesc, error) {
-	mr := MessageReflect(m)
-	if mr == nil || !mr.IsValid() || mr.Descriptor().ExtensionRanges().Len() == 0 {
-		return nil, errNotExtendable
-	}
-
-	// Collect a set of known extension descriptors.
-	extDescs := make(map[protoreflect.FieldNumber]*ExtensionDesc)
-	mr.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
-		if fd.IsExtension() {
-			xt := fd.(protoreflect.ExtensionTypeDescriptor)
-			if xd, ok := xt.Type().(*ExtensionDesc); ok {
-				extDescs[fd.Number()] = xd
-			}
-		}
-		return true
-	})
-
-	// Collect a set of unknown extension descriptors.
-	extRanges := mr.Descriptor().ExtensionRanges()
-	for b := mr.GetUnknown(); len(b) > 0; {
-		num, _, n := protowire.ConsumeField(b)
-		if extRanges.Has(num) && extDescs[num] == nil {
-			extDescs[num] = nil
-		}
-		b = b[n:]
-	}
-
-	// Transpose the set of descriptors into a list.
-	var xts []*ExtensionDesc
-	for num, xt := range extDescs {
-		if xt == nil {
-			xt = &ExtensionDesc{Field: int32(num)}
-		}
-		xts = append(xts, xt)
-	}
-	return xts, nil
-}
-
-// isValidExtension reports whether xtd is a valid extension descriptor for md.
-func isValidExtension(md protoreflect.MessageDescriptor, xtd protoreflect.ExtensionTypeDescriptor) bool {
-	return xtd.ContainingMessage() == md && md.ExtensionRanges().Has(xtd.Number())
-}
-
-// isScalarKind reports whether k is a protobuf scalar kind (except bytes).
-// This function exists for historical reasons since the representation of
-// scalars differs between v1 and v2, where v1 uses *T and v2 uses T.
-func isScalarKind(k reflect.Kind) bool {
-	switch k {
-	case reflect.Bool, reflect.Int32, reflect.Int64, reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64, reflect.String:
-		return true
-	default:
-		return false
-	}
-}
-
-// clearUnknown removes unknown fields from m where remover.Has reports true.
-func clearUnknown(m protoreflect.Message, remover interface {
-	Has(protoreflect.FieldNumber) bool
-}) {
-	var bo protoreflect.RawFields
-	for bi := m.GetUnknown(); len(bi) > 0; {
-		num, _, n := protowire.ConsumeField(bi)
-		if !remover.Has(num) {
-			bo = append(bo, bi[:n]...)
-		}
-		bi = bi[n:]
-	}
-	if bi := m.GetUnknown(); len(bi) != len(bo) {
-		m.SetUnknown(bo)
-	}
-}
-
-type fieldNum protoreflect.FieldNumber
-
-func (n1 fieldNum) Has(n2 protoreflect.FieldNumber) bool {
-	return protoreflect.FieldNumber(n1) == n2
-}

vendor/github.com/golang/protobuf/proto/properties.go

@@ -1,306 +0,0 @@
-// Copyright 2010 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package proto
-
-import (
-	"fmt"
-	"reflect"
-	"strconv"
-	"strings"
-	"sync"
-
-	"google.golang.org/protobuf/reflect/protoreflect"
-	"google.golang.org/protobuf/runtime/protoimpl"
-)
-
-// StructProperties represents protocol buffer type information for a
-// generated protobuf message in the open-struct API.
-//
-// Deprecated: Do not use.
-type StructProperties struct {
-	// Prop are the properties for each field.
-	//
-	// Fields belonging to a oneof are stored in OneofTypes instead, with a
-	// single Properties representing the parent oneof held here.
-	//
-	// The order of Prop matches the order of fields in the Go struct.
-	// Struct fields that are not related to protobufs have a "XXX_" prefix
-	// in the Properties.Name and must be ignored by the user.
-	Prop []*Properties
-
-	// OneofTypes contains information about the oneof fields in this message.
-	// It is keyed by the protobuf field name.
-	OneofTypes map[string]*OneofProperties
-}
-
-// Properties represents the type information for a protobuf message field.
-//
-// Deprecated: Do not use.
-type Properties struct {
-	// Name is a placeholder name with little meaningful semantic value.
-	// If the name has an "XXX_" prefix, the entire Properties must be ignored.
-	Name string
-	// OrigName is the protobuf field name or oneof name.
-	OrigName string
-	// JSONName is the JSON name for the protobuf field.
-	JSONName string
-	// Enum is a placeholder name for enums.
-	// For historical reasons, this is neither the Go name for the enum,
-	// nor the protobuf name for the enum.
-	Enum string // Deprecated: Do not use.
-	// Weak contains the full name of the weakly referenced message.
-	Weak string
-	// Wire is a string representation of the wire type.
-	Wire string
-	// WireType is the protobuf wire type for the field.
-	WireType int
-	// Tag is the protobuf field number.
-	Tag int
-	// Required reports whether this is a required field.
-	Required bool
-	// Optional reports whether this is a optional field.
-	Optional bool
-	// Repeated reports whether this is a repeated field.
-	Repeated bool
-	// Packed reports whether this is a packed repeated field of scalars.
-	Packed bool
-	// Proto3 reports whether this field operates under the proto3 syntax.
-	Proto3 bool
-	// Oneof reports whether this field belongs within a oneof.
-	Oneof bool
-
-	// Default is the default value in string form.
-	Default string
-	// HasDefault reports whether the field has a default value.
-	HasDefault bool
-
-	// MapKeyProp is the properties for the key field for a map field.
-	MapKeyProp *Properties
-	// MapValProp is the properties for the value field for a map field.
-	MapValProp *Properties
-}
-
-// OneofProperties represents the type information for a protobuf oneof.
-//
-// Deprecated: Do not use.
-type OneofProperties struct {
-	// Type is a pointer to the generated wrapper type for the field value.
-	// This is nil for messages that are not in the open-struct API.
-	Type reflect.Type
-	// Field is the index into StructProperties.Prop for the containing oneof.
-	Field int
-	// Prop is the properties for the field.
-	Prop *Properties
-}
-
-// String formats the properties in the protobuf struct field tag style.
-func (p *Properties) String() string {
-	s := p.Wire
-	s += "," + strconv.Itoa(p.Tag)
-	if p.Required {
-		s += ",req"
-	}
-	if p.Optional {
-		s += ",opt"
-	}
-	if p.Repeated {
-		s += ",rep"
-	}
-	if p.Packed {
-		s += ",packed"
-	}
-	s += ",name=" + p.OrigName
-	if p.JSONName != "" {
-		s += ",json=" + p.JSONName
-	}
-	if len(p.Enum) > 0 {
-		s += ",enum=" + p.Enum
-	}
-	if len(p.Weak) > 0 {
-		s += ",weak=" + p.Weak
-	}
-	if p.Proto3 {
-		s += ",proto3"
-	}
-	if p.Oneof {
-		s += ",oneof"
-	}
-	if p.HasDefault {
-		s += ",def=" + p.Default
-	}
-	return s
-}
-
-// Parse populates p by parsing a string in the protobuf struct field tag style.
-func (p *Properties) Parse(tag string) {
-	// For example: "bytes,49,opt,name=foo,def=hello!"
-	for len(tag) > 0 {
-		i := strings.IndexByte(tag, ',')
-		if i < 0 {
-			i = len(tag)
-		}
-		switch s := tag[:i]; {
-		case strings.HasPrefix(s, "name="):
-			p.OrigName = s[len("name="):]
-		case strings.HasPrefix(s, "json="):
-			p.JSONName = s[len("json="):]
-		case strings.HasPrefix(s, "enum="):
-			p.Enum = s[len("enum="):]
-		case strings.HasPrefix(s, "weak="):
-			p.Weak = s[len("weak="):]
-		case strings.Trim(s, "0123456789") == "":
-			n, _ := strconv.ParseUint(s, 10, 32)
-			p.Tag = int(n)
-		case s == "opt":
-			p.Optional = true
-		case s == "req":
-			p.Required = true
-		case s == "rep":
-			p.Repeated = true
-		case s == "varint" || s == "zigzag32" || s == "zigzag64":
-			p.Wire = s
-			p.WireType = WireVarint
-		case s == "fixed32":
-			p.Wire = s
-			p.WireType = WireFixed32
-		case s == "fixed64":
-			p.Wire = s
-			p.WireType = WireFixed64
-		case s == "bytes":
-			p.Wire = s
-			p.WireType = WireBytes
-		case s == "group":
-			p.Wire = s
-			p.WireType = WireStartGroup
-		case s == "packed":
-			p.Packed = true
-		case s == "proto3":
-			p.Proto3 = true
-		case s == "oneof":
-			p.Oneof = true
-		case strings.HasPrefix(s, "def="):
-			// The default tag is special in that everything afterwards is the
-			// default regardless of the presence of commas.
-			p.HasDefault = true
-			p.Default, i = tag[len("def="):], len(tag)
-		}
-		tag = strings.TrimPrefix(tag[i:], ",")
-	}
-}
-
-// Init populates the properties from a protocol buffer struct tag.
-//
-// Deprecated: Do not use.
-func (p *Properties) Init(typ reflect.Type, name, tag string, f *reflect.StructField) {
-	p.Name = name
-	p.OrigName = name
-	if tag == "" {
-		return
-	}
-	p.Parse(tag)
-
-	if typ != nil && typ.Kind() == reflect.Map {
-		p.MapKeyProp = new(Properties)
-		p.MapKeyProp.Init(nil, "Key", f.Tag.Get("protobuf_key"), nil)
-		p.MapValProp = new(Properties)
-		p.MapValProp.Init(nil, "Value", f.Tag.Get("protobuf_val"), nil)
-	}
-}
-
-var propertiesCache sync.Map // map[reflect.Type]*StructProperties
-
-// GetProperties returns the list of properties for the type represented by t,
-// which must be a generated protocol buffer message in the open-struct API,
-// where protobuf message fields are represented by exported Go struct fields.
-//
-// Deprecated: Use protobuf reflection instead.
-func GetProperties(t reflect.Type) *StructProperties {
-	if p, ok := propertiesCache.Load(t); ok {
-		return p.(*StructProperties)
-	}
-	p, _ := propertiesCache.LoadOrStore(t, newProperties(t))
-	return p.(*StructProperties)
-}
-
-func newProperties(t reflect.Type) *StructProperties {
-	if t.Kind() != reflect.Struct {
-		panic(fmt.Sprintf("%v is not a generated message in the open-struct API", t))
-	}
-
-	var hasOneof bool
-	prop := new(StructProperties)
-
-	// Construct a list of properties for each field in the struct.
-	for i := 0; i < t.NumField(); i++ {
-		p := new(Properties)
-		f := t.Field(i)
-		tagField := f.Tag.Get("protobuf")
-		p.Init(f.Type, f.Name, tagField, &f)
-
-		tagOneof := f.Tag.Get("protobuf_oneof")
-		if tagOneof != "" {
-			hasOneof = true
-			p.OrigName = tagOneof
-		}
-
-		// Rename unrelated struct fields with the "XXX_" prefix since so much
-		// user code simply checks for this to exclude special fields.
-		if tagField == "" && tagOneof == "" && !strings.HasPrefix(p.Name, "XXX_") {
-			p.Name = "XXX_" + p.Name
-			p.OrigName = "XXX_" + p.OrigName
-		} else if p.Weak != "" {
-			p.Name = p.OrigName // avoid possible "XXX_" prefix on weak field
-		}
-
-		prop.Prop = append(prop.Prop, p)
-	}
-
-	// Construct a mapping of oneof field names to properties.
-	if hasOneof {
-		var oneofWrappers []interface{}
-		if fn, ok := reflect.PtrTo(t).MethodByName("XXX_OneofFuncs"); ok {
-			oneofWrappers = fn.Func.Call([]reflect.Value{reflect.Zero(fn.Type.In(0))})[3].Interface().([]interface{})
-		}
-		if fn, ok := reflect.PtrTo(t).MethodByName("XXX_OneofWrappers"); ok {
-			oneofWrappers = fn.Func.Call([]reflect.Value{reflect.Zero(fn.Type.In(0))})[0].Interface().([]interface{})
-		}
-		if m, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(protoreflect.ProtoMessage); ok {
-			if m, ok := m.ProtoReflect().(interface{ ProtoMessageInfo() *protoimpl.MessageInfo }); ok {
-				oneofWrappers = m.ProtoMessageInfo().OneofWrappers
-			}
-		}
-
-		prop.OneofTypes = make(map[string]*OneofProperties)
-		for _, wrapper := range oneofWrappers {
-			p := &OneofProperties{
-				Type: reflect.ValueOf(wrapper).Type(), // *T
-				Prop: new(Properties),
-			}
-			f := p.Type.Elem().Field(0)
-			p.Prop.Name = f.Name
-			p.Prop.Parse(f.Tag.Get("protobuf"))
-
-			// Determine the struct field that contains this oneof.
-			// Each wrapper is assignable to exactly one parent field.
-			var foundOneof bool
-			for i := 0; i < t.NumField() && !foundOneof; i++ {
-				if p.Type.AssignableTo(t.Field(i).Type) {
-					p.Field = i
-					foundOneof = true
-				}
-			}
-			if !foundOneof {
-				panic(fmt.Sprintf("%v is not a generated message in the open-struct API", t))
-			}
-			prop.OneofTypes[p.Prop.OrigName] = p
-		}
-	}
-
-	return prop
-}
-
-func (sp *StructProperties) Len() int           { return len(sp.Prop) }
-func (sp *StructProperties) Less(i, j int) bool { return false }
-func (sp *StructProperties) Swap(i, j int)      { return }

vendor/github.com/golang/protobuf/proto/proto.go

@@ -1,167 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package proto provides functionality for handling protocol buffer messages.
-// In particular, it provides marshaling and unmarshaling between a protobuf
-// message and the binary wire format.
-//
-// See https://developers.google.com/protocol-buffers/docs/gotutorial for
-// more information.
-//
-// Deprecated: Use the "google.golang.org/protobuf/proto" package instead.
-package proto
-
-import (
-	protoV2 "google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/reflect/protoreflect"
-	"google.golang.org/protobuf/runtime/protoiface"
-	"google.golang.org/protobuf/runtime/protoimpl"
-)
-
-const (
-	ProtoPackageIsVersion1 = true
-	ProtoPackageIsVersion2 = true
-	ProtoPackageIsVersion3 = true
-	ProtoPackageIsVersion4 = true
-)
-
-// GeneratedEnum is any enum type generated by protoc-gen-go
-// which is a named int32 kind.
-// This type exists for documentation purposes.
-type GeneratedEnum interface{}
-
-// GeneratedMessage is any message type generated by protoc-gen-go
-// which is a pointer to a named struct kind.
-// This type exists for documentation purposes.
-type GeneratedMessage interface{}
-
-// Message is a protocol buffer message.
-//
-// This is the v1 version of the message interface and is marginally better
-// than an empty interface as it lacks any method to programatically interact
-// with the contents of the message.
-//
-// A v2 message is declared in "google.golang.org/protobuf/proto".Message and
-// exposes protobuf reflection as a first-class feature of the interface.
-//
-// To convert a v1 message to a v2 message, use the MessageV2 function.
-// To convert a v2 message to a v1 message, use the MessageV1 function.
-type Message = protoiface.MessageV1
-
-// MessageV1 converts either a v1 or v2 message to a v1 message.
-// It returns nil if m is nil.
-func MessageV1(m GeneratedMessage) protoiface.MessageV1 {
-	return protoimpl.X.ProtoMessageV1Of(m)
-}
-
-// MessageV2 converts either a v1 or v2 message to a v2 message.
-// It returns nil if m is nil.
-func MessageV2(m GeneratedMessage) protoV2.Message {
-	return protoimpl.X.ProtoMessageV2Of(m)
-}
-
-// MessageReflect returns a reflective view for a message.
-// It returns nil if m is nil.
-func MessageReflect(m Message) protoreflect.Message {
-	return protoimpl.X.MessageOf(m)
-}
-
-// Marshaler is implemented by messages that can marshal themselves.
-// This interface is used by the following functions: Size, Marshal,
-// Buffer.Marshal, and Buffer.EncodeMessage.
-//
-// Deprecated: Do not implement.
-type Marshaler interface {
-	// Marshal formats the encoded bytes of the message.
-	// It should be deterministic and emit valid protobuf wire data.
-	// The caller takes ownership of the returned buffer.
-	Marshal() ([]byte, error)
-}
-
-// Unmarshaler is implemented by messages that can unmarshal themselves.
-// This interface is used by the following functions: Unmarshal, UnmarshalMerge,
-// Buffer.Unmarshal, Buffer.DecodeMessage, and Buffer.DecodeGroup.
-//
-// Deprecated: Do not implement.
-type Unmarshaler interface {
-	// Unmarshal parses the encoded bytes of the protobuf wire input.
-	// The provided buffer is only valid for during method call.
-	// It should not reset the receiver message.
-	Unmarshal([]byte) error
-}
-
-// Merger is implemented by messages that can merge themselves.
-// This interface is used by the following functions: Clone and Merge.
-//
-// Deprecated: Do not implement.
-type Merger interface {
-	// Merge merges the contents of src into the receiver message.
-	// It clones all data structures in src such that it aliases no mutable
-	// memory referenced by src.
-	Merge(src Message)
-}
-
-// RequiredNotSetError is an error type returned when
-// marshaling or unmarshaling a message with missing required fields.
-type RequiredNotSetError struct {
-	err error
-}
-
-func (e *RequiredNotSetError) Error() string {
-	if e.err != nil {
-		return e.err.Error()
-	}
-	return "proto: required field not set"
-}
-func (e *RequiredNotSetError) RequiredNotSet() bool {
-	return true
-}
-
-func checkRequiredNotSet(m protoV2.Message) error {
-	if err := protoV2.CheckInitialized(m); err != nil {
-		return &RequiredNotSetError{err: err}
-	}
-	return nil
-}
-
-// Clone returns a deep copy of src.
-func Clone(src Message) Message {
-	return MessageV1(protoV2.Clone(MessageV2(src)))
-}
-
-// Merge merges src into dst, which must be messages of the same type.
-//
-// Populated scalar fields in src are copied to dst, while populated
-// singular messages in src are merged into dst by recursively calling Merge.
-// The elements of every list field in src is appended to the corresponded
-// list fields in dst. The entries of every map field in src is copied into
-// the corresponding map field in dst, possibly replacing existing entries.
-// The unknown fields of src are appended to the unknown fields of dst.
-func Merge(dst, src Message) {
-	protoV2.Merge(MessageV2(dst), MessageV2(src))
-}
-
-// Equal reports whether two messages are equal.
-// If two messages marshal to the same bytes under deterministic serialization,
-// then Equal is guaranteed to report true.
-//
-// Two messages are equal if they are the same protobuf message type,
-// have the same set of populated known and extension field values,
-// and the same set of unknown fields values.
-//
-// Scalar values are compared with the equivalent of the == operator in Go,
-// except bytes values which are compared using bytes.Equal and
-// floating point values which specially treat NaNs as equal.
-// Message values are compared by recursively calling Equal.
-// Lists are equal if each element value is also equal.
-// Maps are equal if they have the same set of keys, where the pair of values
-// for each key is also equal.
-func Equal(x, y Message) bool {
-	return protoV2.Equal(MessageV2(x), MessageV2(y))
-}
-
-func isMessageSet(md protoreflect.MessageDescriptor) bool {
-	ms, ok := md.(interface{ IsMessageSet() bool })
-	return ok && ms.IsMessageSet()
-}

vendor/github.com/golang/protobuf/proto/registry.go

@@ -1,317 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package proto
-
-import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"io/ioutil"
-	"reflect"
-	"strings"
-	"sync"
-
-	"google.golang.org/protobuf/reflect/protodesc"
-	"google.golang.org/protobuf/reflect/protoreflect"
-	"google.golang.org/protobuf/reflect/protoregistry"
-	"google.golang.org/protobuf/runtime/protoimpl"
-)
-
-// filePath is the path to the proto source file.
-type filePath = string // e.g., "google/protobuf/descriptor.proto"
-
-// fileDescGZIP is the compressed contents of the encoded FileDescriptorProto.
-type fileDescGZIP = []byte
-
-var fileCache sync.Map // map[filePath]fileDescGZIP
-
-// RegisterFile is called from generated code to register the compressed
-// FileDescriptorProto with the file path for a proto source file.
-//
-// Deprecated: Use protoregistry.GlobalFiles.RegisterFile instead.
-func RegisterFile(s filePath, d fileDescGZIP) {
-	// Decompress the descriptor.
-	zr, err := gzip.NewReader(bytes.NewReader(d))
-	if err != nil {
-		panic(fmt.Sprintf("proto: invalid compressed file descriptor: %v", err))
-	}
-	b, err := ioutil.ReadAll(zr)
-	if err != nil {
-		panic(fmt.Sprintf("proto: invalid compressed file descriptor: %v", err))
-	}
-
-	// Construct a protoreflect.FileDescriptor from the raw descriptor.
-	// Note that DescBuilder.Build automatically registers the constructed
-	// file descriptor with the v2 registry.
-	protoimpl.DescBuilder{RawDescriptor: b}.Build()
-
-	// Locally cache the raw descriptor form for the file.
-	fileCache.Store(s, d)
-}
-
-// FileDescriptor returns the compressed FileDescriptorProto given the file path
-// for a proto source file. It returns nil if not found.
-//
-// Deprecated: Use protoregistry.GlobalFiles.FindFileByPath instead.
-func FileDescriptor(s filePath) fileDescGZIP {
-	if v, ok := fileCache.Load(s); ok {
-		return v.(fileDescGZIP)
-	}
-
-	// Find the descriptor in the v2 registry.
-	var b []byte
-	if fd, _ := protoregistry.GlobalFiles.FindFileByPath(s); fd != nil {
-		b, _ = Marshal(protodesc.ToFileDescriptorProto(fd))
-	}
-
-	// Locally cache the raw descriptor form for the file.
-	if len(b) > 0 {
-		v, _ := fileCache.LoadOrStore(s, protoimpl.X.CompressGZIP(b))
-		return v.(fileDescGZIP)
-	}
-	return nil
-}
-
-// enumName is the name of an enum. For historical reasons, the enum name is
-// neither the full Go name nor the full protobuf name of the enum.
-// The name is the dot-separated combination of just the proto package that the
-// enum is declared within followed by the Go type name of the generated enum.
-type enumName = string // e.g., "my.proto.package.GoMessage_GoEnum"
-
-// enumsByName maps enum values by name to their numeric counterpart.
-type enumsByName = map[string]int32
-
-// enumsByNumber maps enum values by number to their name counterpart.
-type enumsByNumber = map[int32]string
-
-var enumCache sync.Map     // map[enumName]enumsByName
-var numFilesCache sync.Map // map[protoreflect.FullName]int
-
-// RegisterEnum is called from the generated code to register the mapping of
-// enum value names to enum numbers for the enum identified by s.
-//
-// Deprecated: Use protoregistry.GlobalTypes.RegisterEnum instead.
-func RegisterEnum(s enumName, _ enumsByNumber, m enumsByName) {
-	if _, ok := enumCache.Load(s); ok {
-		panic("proto: duplicate enum registered: " + s)
-	}
-	enumCache.Store(s, m)
-
-	// This does not forward registration to the v2 registry since this API
-	// lacks sufficient information to construct a complete v2 enum descriptor.
-}
-
-// EnumValueMap returns the mapping from enum value names to enum numbers for
-// the enum of the given name. It returns nil if not found.
-//
-// Deprecated: Use protoregistry.GlobalTypes.FindEnumByName instead.
-func EnumValueMap(s enumName) enumsByName {
-	if v, ok := enumCache.Load(s); ok {
-		return v.(enumsByName)
-	}
-
-	// Check whether the cache is stale. If the number of files in the current
-	// package differs, then it means that some enums may have been recently
-	// registered upstream that we do not know about.
-	var protoPkg protoreflect.FullName
-	if i := strings.LastIndexByte(s, '.'); i >= 0 {
-		protoPkg = protoreflect.FullName(s[:i])
-	}
-	v, _ := numFilesCache.Load(protoPkg)
-	numFiles, _ := v.(int)
-	if protoregistry.GlobalFiles.NumFilesByPackage(protoPkg) == numFiles {
-		return nil // cache is up-to-date; was not found earlier
-	}
-
-	// Update the enum cache for all enums declared in the given proto package.
-	numFiles = 0
-	protoregistry.GlobalFiles.RangeFilesByPackage(protoPkg, func(fd protoreflect.FileDescriptor) bool {
-		walkEnums(fd, func(ed protoreflect.EnumDescriptor) {
-			name := protoimpl.X.LegacyEnumName(ed)
-			if _, ok := enumCache.Load(name); !ok {
-				m := make(enumsByName)
-				evs := ed.Values()
-				for i := evs.Len() - 1; i >= 0; i-- {
-					ev := evs.Get(i)
-					m[string(ev.Name())] = int32(ev.Number())
-				}
-				enumCache.LoadOrStore(name, m)
-			}
-		})
-		numFiles++
-		return true
-	})
-	numFilesCache.Store(protoPkg, numFiles)
-
-	// Check cache again for enum map.
-	if v, ok := enumCache.Load(s); ok {
-		return v.(enumsByName)
-	}
-	return nil
-}
-
-// walkEnums recursively walks all enums declared in d.
-func walkEnums(d interface {
-	Enums() protoreflect.EnumDescriptors
-	Messages() protoreflect.MessageDescriptors
-}, f func(protoreflect.EnumDescriptor)) {
-	eds := d.Enums()
-	for i := eds.Len() - 1; i >= 0; i-- {
-		f(eds.Get(i))
-	}
-	mds := d.Messages()
-	for i := mds.Len() - 1; i >= 0; i-- {
-		walkEnums(mds.Get(i), f)
-	}
-}
-
-// messageName is the full name of protobuf message.
-type messageName = string
-
-var messageTypeCache sync.Map // map[messageName]reflect.Type
-
-// RegisterType is called from generated code to register the message Go type
-// for a message of the given name.
-//
-// Deprecated: Use protoregistry.GlobalTypes.RegisterMessage instead.
-func RegisterType(m Message, s messageName) {
-	mt := protoimpl.X.LegacyMessageTypeOf(m, protoreflect.FullName(s))
-	if err := protoregistry.GlobalTypes.RegisterMessage(mt); err != nil {
-		panic(err)
-	}
-	messageTypeCache.Store(s, reflect.TypeOf(m))
-}
-
-// RegisterMapType is called from generated code to register the Go map type
-// for a protobuf message representing a map entry.
-//
-// Deprecated: Do not use.
-func RegisterMapType(m interface{}, s messageName) {
-	t := reflect.TypeOf(m)
-	if t.Kind() != reflect.Map {
-		panic(fmt.Sprintf("invalid map kind: %v", t))
-	}
-	if _, ok := messageTypeCache.Load(s); ok {
-		panic(fmt.Errorf("proto: duplicate proto message registered: %s", s))
-	}
-	messageTypeCache.Store(s, t)
-}
-
-// MessageType returns the message type for a named message.
-// It returns nil if not found.
-//
-// Deprecated: Use protoregistry.GlobalTypes.FindMessageByName instead.
-func MessageType(s messageName) reflect.Type {
-	if v, ok := messageTypeCache.Load(s); ok {
-		return v.(reflect.Type)
-	}
-
-	// Derive the message type from the v2 registry.
-	var t reflect.Type
-	if mt, _ := protoregistry.GlobalTypes.FindMessageByName(protoreflect.FullName(s)); mt != nil {
-		t = messageGoType(mt)
-	}
-
-	// If we could not get a concrete type, it is possible that it is a
-	// pseudo-message for a map entry.
-	if t == nil {
-		d, _ := protoregistry.GlobalFiles.FindDescriptorByName(protoreflect.FullName(s))
-		if md, _ := d.(protoreflect.MessageDescriptor); md != nil && md.IsMapEntry() {
-			kt := goTypeForField(md.Fields().ByNumber(1))
-			vt := goTypeForField(md.Fields().ByNumber(2))
-			t = reflect.MapOf(kt, vt)
-		}
-	}
-
-	// Locally cache the message type for the given name.
-	if t != nil {
-		v, _ := messageTypeCache.LoadOrStore(s, t)
-		return v.(reflect.Type)
-	}
-	return nil
-}
-
-func goTypeForField(fd protoreflect.FieldDescriptor) reflect.Type {
-	switch k := fd.Kind(); k {
-	case protoreflect.EnumKind:
-		if et, _ := protoregistry.GlobalTypes.FindEnumByName(fd.Enum().FullName()); et != nil {
-			return enumGoType(et)
-		}
-		return reflect.TypeOf(protoreflect.EnumNumber(0))
-	case protoreflect.MessageKind, protoreflect.GroupKind:
-		if mt, _ := protoregistry.GlobalTypes.FindMessageByName(fd.Message().FullName()); mt != nil {
-			return messageGoType(mt)
-		}
-		return reflect.TypeOf((*protoreflect.Message)(nil)).Elem()
-	default:
-		return reflect.TypeOf(fd.Default().Interface())
-	}
-}
-
-func enumGoType(et protoreflect.EnumType) reflect.Type {
-	return reflect.TypeOf(et.New(0))
-}
-
-func messageGoType(mt protoreflect.MessageType) reflect.Type {
-	return reflect.TypeOf(MessageV1(mt.Zero().Interface()))
-}
-
-// MessageName returns the full protobuf name for the given message type.
-//
-// Deprecated: Use protoreflect.MessageDescriptor.FullName instead.
-func MessageName(m Message) messageName {
-	if m == nil {
-		return ""
-	}
-	if m, ok := m.(interface{ XXX_MessageName() messageName }); ok {
-		return m.XXX_MessageName()
-	}
-	return messageName(protoimpl.X.MessageDescriptorOf(m).FullName())
-}
-
-// RegisterExtension is called from the generated code to register
-// the extension descriptor.
-//
-// Deprecated: Use protoregistry.GlobalTypes.RegisterExtension instead.
-func RegisterExtension(d *ExtensionDesc) {
-	if err := protoregistry.GlobalTypes.RegisterExtension(d); err != nil {
-		panic(err)
-	}
-}
-
-type extensionsByNumber = map[int32]*ExtensionDesc
-
-var extensionCache sync.Map // map[messageName]extensionsByNumber
-
-// RegisteredExtensions returns a map of the registered extensions for the
-// provided protobuf message, indexed by the extension field number.
-//
-// Deprecated: Use protoregistry.GlobalTypes.RangeExtensionsByMessage instead.
-func RegisteredExtensions(m Message) extensionsByNumber {
-	// Check whether the cache is stale. If the number of extensions for
-	// the given message differs, then it means that some extensions were
-	// recently registered upstream that we do not know about.
-	s := MessageName(m)
-	v, _ := extensionCache.Load(s)
-	xs, _ := v.(extensionsByNumber)
-	if protoregistry.GlobalTypes.NumExtensionsByMessage(protoreflect.FullName(s)) == len(xs) {
-		return xs // cache is up-to-date
-	}
-
-	// Cache is stale, re-compute the extensions map.
-	xs = make(extensionsByNumber)
-	protoregistry.GlobalTypes.RangeExtensionsByMessage(protoreflect.FullName(s), func(xt protoreflect.ExtensionType) bool {
-		if xd, ok := xt.(*ExtensionDesc); ok {
-			xs[int32(xt.TypeDescriptor().Number())] = xd
-		} else {
-			// TODO: This implies that the protoreflect.ExtensionType is a
-			// custom type not generated by protoc-gen-go. We could try and
-			// convert the type to an ExtensionDesc.
-		}
-		return true
-	})
-	extensionCache.Store(s, xs)
-	return xs
-}

vendor/github.com/golang/protobuf/proto/text_decode.go

@@ -1,801 +0,0 @@
-// Copyright 2010 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package proto
-
-import (
-	"encoding"
-	"errors"
-	"fmt"
-	"reflect"
-	"strconv"
-	"strings"
-	"unicode/utf8"
-
-	"google.golang.org/protobuf/encoding/prototext"
-	protoV2 "google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/reflect/protoreflect"
-	"google.golang.org/protobuf/reflect/protoregistry"
-)
-
-const wrapTextUnmarshalV2 = false
-
-// ParseError is returned by UnmarshalText.
-type ParseError struct {
-	Message string
-
-	// Deprecated: Do not use.
-	Line, Offset int
-}
-
-func (e *ParseError) Error() string {
-	if wrapTextUnmarshalV2 {
-		return e.Message
-	}
-	if e.Line == 1 {
-		return fmt.Sprintf("line 1.%d: %v", e.Offset, e.Message)
-	}
-	return fmt.Sprintf("line %d: %v", e.Line, e.Message)
-}
-
-// UnmarshalText parses a proto text formatted string into m.
-func UnmarshalText(s string, m Message) error {
-	if u, ok := m.(encoding.TextUnmarshaler); ok {
-		return u.UnmarshalText([]byte(s))
-	}
-
-	m.Reset()
-	mi := MessageV2(m)
-
-	if wrapTextUnmarshalV2 {
-		err := prototext.UnmarshalOptions{
-			AllowPartial: true,
-		}.Unmarshal([]byte(s), mi)
-		if err != nil {
-			return &ParseError{Message: err.Error()}
-		}
-		return checkRequiredNotSet(mi)
-	} else {
-		if err := newTextParser(s).unmarshalMessage(mi.ProtoReflect(), ""); err != nil {
-			return err
-		}
-		return checkRequiredNotSet(mi)
-	}
-}
-
-type textParser struct {
-	s            string // remaining input
-	done         bool   // whether the parsing is finished (success or error)
-	backed       bool   // whether back() was called
-	offset, line int
-	cur          token
-}
-
-type token struct {
-	value    string
-	err      *ParseError
-	line     int    // line number
-	offset   int    // byte number from start of input, not start of line
-	unquoted string // the unquoted version of value, if it was a quoted string
-}
-
-func newTextParser(s string) *textParser {
-	p := new(textParser)
-	p.s = s
-	p.line = 1
-	p.cur.line = 1
-	return p
-}
-
-func (p *textParser) unmarshalMessage(m protoreflect.Message, terminator string) (err error) {
-	md := m.Descriptor()
-	fds := md.Fields()
-
-	// A struct is a sequence of "name: value", terminated by one of
-	// '>' or '}', or the end of the input.  A name may also be
-	// "[extension]" or "[type/url]".
-	//
-	// The whole struct can also be an expanded Any message, like:
-	// [type/url] < ... struct contents ... >
-	seen := make(map[protoreflect.FieldNumber]bool)
-	for {
-		tok := p.next()
-		if tok.err != nil {
-			return tok.err
-		}
-		if tok.value == terminator {
-			break
-		}
-		if tok.value == "[" {
-			if err := p.unmarshalExtensionOrAny(m, seen); err != nil {
-				return err
-			}
-			continue
-		}
-
-		// This is a normal, non-extension field.
-		name := protoreflect.Name(tok.value)
-		fd := fds.ByName(name)
-		switch {
-		case fd == nil:
-			gd := fds.ByName(protoreflect.Name(strings.ToLower(string(name))))
-			if gd != nil && gd.Kind() == protoreflect.GroupKind && gd.Message().Name() == name {
-				fd = gd
-			}
-		case fd.Kind() == protoreflect.GroupKind && fd.Message().Name() != name:
-			fd = nil
-		case fd.IsWeak() && fd.Message().IsPlaceholder():
-			fd = nil
-		}
-		if fd == nil {
-			typeName := string(md.FullName())
-			if m, ok := m.Interface().(Message); ok {
-				t := reflect.TypeOf(m)
-				if t.Kind() == reflect.Ptr {
-					typeName = t.Elem().String()
-				}
-			}
-			return p.errorf("unknown field name %q in %v", name, typeName)
-		}
-		if od := fd.ContainingOneof(); od != nil && m.WhichOneof(od) != nil {
-			return p.errorf("field '%s' would overwrite already parsed oneof '%s'", name, od.Name())
-		}
-		if fd.Cardinality() != protoreflect.Repeated && seen[fd.Number()] {
-			return p.errorf("non-repeated field %q was repeated", fd.Name())
-		}
-		seen[fd.Number()] = true
-
-		// Consume any colon.
-		if err := p.checkForColon(fd); err != nil {
-			return err
-		}
-
-		// Parse into the field.
-		v := m.Get(fd)
-		if !m.Has(fd) && (fd.IsList() || fd.IsMap() || fd.Message() != nil) {
-			v = m.Mutable(fd)
-		}
-		if v, err = p.unmarshalValue(v, fd); err != nil {
-			return err
-		}
-		m.Set(fd, v)
-
-		if err := p.consumeOptionalSeparator(); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (p *textParser) unmarshalExtensionOrAny(m protoreflect.Message, seen map[protoreflect.FieldNumber]bool) error {
-	name, err := p.consumeExtensionOrAnyName()
-	if err != nil {
-		return err
-	}
-
-	// If it contains a slash, it's an Any type URL.
-	if slashIdx := strings.LastIndex(name, "/"); slashIdx >= 0 {
-		tok := p.next()
-		if tok.err != nil {
-			return tok.err
-		}
-		// consume an optional colon
-		if tok.value == ":" {
-			tok = p.next()
-			if tok.err != nil {
-				return tok.err
-			}
-		}
-
-		var terminator string
-		switch tok.value {
-		case "<":
-			terminator = ">"
-		case "{":
-			terminator = "}"
-		default:
-			return p.errorf("expected '{' or '<', found %q", tok.value)
-		}
-
-		mt, err := protoregistry.GlobalTypes.FindMessageByURL(name)
-		if err != nil {
-			return p.errorf("unrecognized message %q in google.protobuf.Any", name[slashIdx+len("/"):])
-		}
-		m2 := mt.New()
-		if err := p.unmarshalMessage(m2, terminator); err != nil {
-			return err
-		}
-		b, err := protoV2.Marshal(m2.Interface())
-		if err != nil {
-			return p.errorf("failed to marshal message of type %q: %v", name[slashIdx+len("/"):], err)
-		}
-
-		urlFD := m.Descriptor().Fields().ByName("type_url")
-		valFD := m.Descriptor().Fields().ByName("value")
-		if seen[urlFD.Number()] {
-			return p.errorf("Any message unpacked multiple times, or %q already set", urlFD.Name())
-		}
-		if seen[valFD.Number()] {
-			return p.errorf("Any message unpacked multiple times, or %q already set", valFD.Name())
-		}
-		m.Set(urlFD, protoreflect.ValueOfString(name))
-		m.Set(valFD, protoreflect.ValueOfBytes(b))
-		seen[urlFD.Number()] = true
-		seen[valFD.Number()] = true
-		return nil
-	}
-
-	xname := protoreflect.FullName(name)
-	xt, _ := protoregistry.GlobalTypes.FindExtensionByName(xname)
-	if xt == nil && isMessageSet(m.Descriptor()) {
-		xt, _ = protoregistry.GlobalTypes.FindExtensionByName(xname.Append("message_set_extension"))
-	}
-	if xt == nil {
-		return p.errorf("unrecognized extension %q", name)
-	}
-	fd := xt.TypeDescriptor()
-	if fd.ContainingMessage().FullName() != m.Descriptor().FullName() {
-		return p.errorf("extension field %q does not extend message %q", name, m.Descriptor().FullName())
-	}
-
-	if err := p.checkForColon(fd); err != nil {
-		return err
-	}
-
-	v := m.Get(fd)
-	if !m.Has(fd) && (fd.IsList() || fd.IsMap() || fd.Message() != nil) {
-		v = m.Mutable(fd)
-	}
-	v, err = p.unmarshalValue(v, fd)
-	if err != nil {
-		return err
-	}
-	m.Set(fd, v)
-	return p.consumeOptionalSeparator()
-}
-
-func (p *textParser) unmarshalValue(v protoreflect.Value, fd protoreflect.FieldDescriptor) (protoreflect.Value, error) {
-	tok := p.next()
-	if tok.err != nil {
-		return v, tok.err
-	}
-	if tok.value == "" {
-		return v, p.errorf("unexpected EOF")
-	}
-
-	switch {
-	case fd.IsList():
-		lv := v.List()
-		var err error
-		if tok.value == "[" {
-			// Repeated field with list notation, like [1,2,3].
-			for {
-				vv := lv.NewElement()
-				vv, err = p.unmarshalSingularValue(vv, fd)
-				if err != nil {
-					return v, err
-				}
-				lv.Append(vv)
-
-				tok := p.next()
-				if tok.err != nil {
-					return v, tok.err
-				}
-				if tok.value == "]" {
-					break
-				}
-				if tok.value != "," {
-					return v, p.errorf("Expected ']' or ',' found %q", tok.value)
-				}
-			}
-			return v, nil
-		}
-
-		// One value of the repeated field.
-		p.back()
-		vv := lv.NewElement()
-		vv, err = p.unmarshalSingularValue(vv, fd)
-		if err != nil {
-			return v, err
-		}
-		lv.Append(vv)
-		return v, nil
-	case fd.IsMap():
-		// The map entry should be this sequence of tokens:
-		//	< key : KEY value : VALUE >
-		// However, implementations may omit key or value, and technically
-		// we should support them in any order.
-		var terminator string
-		switch tok.value {
-		case "<":
-			terminator = ">"
-		case "{":
-			terminator = "}"
-		default:
-			return v, p.errorf("expected '{' or '<', found %q", tok.value)
-		}
-
-		keyFD := fd.MapKey()
-		valFD := fd.MapValue()
-
-		mv := v.Map()
-		kv := keyFD.Default()
-		vv := mv.NewValue()
-		for {
-			tok := p.next()
-			if tok.err != nil {
-				return v, tok.err
-			}
-			if tok.value == terminator {
-				break
-			}
-			var err error
-			switch tok.value {
-			case "key":
-				if err := p.consumeToken(":"); err != nil {
-					return v, err
-				}
-				if kv, err = p.unmarshalSingularValue(kv, keyFD); err != nil {
-					return v, err
-				}
-				if err := p.consumeOptionalSeparator(); err != nil {
-					return v, err
-				}
-			case "value":
-				if err := p.checkForColon(valFD); err != nil {
-					return v, err
-				}
-				if vv, err = p.unmarshalSingularValue(vv, valFD); err != nil {
-					return v, err
-				}
-				if err := p.consumeOptionalSeparator(); err != nil {
-					return v, err
-				}
-			default:
-				p.back()
-				return v, p.errorf(`expected "key", "value", or %q, found %q`, terminator, tok.value)
-			}
-		}
-		mv.Set(kv.MapKey(), vv)
-		return v, nil
-	default:
-		p.back()
-		return p.unmarshalSingularValue(v, fd)
-	}
-}
-
-func (p *textParser) unmarshalSingularValue(v protoreflect.Value, fd protoreflect.FieldDescriptor) (protoreflect.Value, error) {
-	tok := p.next()
-	if tok.err != nil {
-		return v, tok.err
-	}
-	if tok.value == "" {
-		return v, p.errorf("unexpected EOF")
-	}
-
-	switch fd.Kind() {
-	case protoreflect.BoolKind:
-		switch tok.value {
-		case "true", "1", "t", "True":
-			return protoreflect.ValueOfBool(true), nil
-		case "false", "0", "f", "False":
-			return protoreflect.ValueOfBool(false), nil
-		}
-	case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind:
-		if x, err := strconv.ParseInt(tok.value, 0, 32); err == nil {
-			return protoreflect.ValueOfInt32(int32(x)), nil
-		}
-
-		// The C++ parser accepts large positive hex numbers that uses
-		// two's complement arithmetic to represent negative numbers.
-		// This feature is here for backwards compatibility with C++.
-		if strings.HasPrefix(tok.value, "0x") {
-			if x, err := strconv.ParseUint(tok.value, 0, 32); err == nil {
-				return protoreflect.ValueOfInt32(int32(-(int64(^x) + 1))), nil
-			}
-		}
-	case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
-		if x, err := strconv.ParseInt(tok.value, 0, 64); err == nil {
-			return protoreflect.ValueOfInt64(int64(x)), nil
-		}
-
-		// The C++ parser accepts large positive hex numbers that uses
-		// two's complement arithmetic to represent negative numbers.
-		// This feature is here for backwards compatibility with C++.
-		if strings.HasPrefix(tok.value, "0x") {
-			if x, err := strconv.ParseUint(tok.value, 0, 64); err == nil {
-				return protoreflect.ValueOfInt64(int64(-(int64(^x) + 1))), nil
-			}
-		}
-	case protoreflect.Uint32Kind, protoreflect.Fixed32Kind:
-		if x, err := strconv.ParseUint(tok.value, 0, 32); err == nil {
-			return protoreflect.ValueOfUint32(uint32(x)), nil
-		}
-	case protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
-		if x, err := strconv.ParseUint(tok.value, 0, 64); err == nil {
-			return protoreflect.ValueOfUint64(uint64(x)), nil
-		}
-	case protoreflect.FloatKind:
-		// Ignore 'f' for compatibility with output generated by C++,
-		// but don't remove 'f' when the value is "-inf" or "inf".
-		v := tok.value
-		if strings.HasSuffix(v, "f") && v != "-inf" && v != "inf" {
-			v = v[:len(v)-len("f")]
-		}
-		if x, err := strconv.ParseFloat(v, 32); err == nil {
-			return protoreflect.ValueOfFloat32(float32(x)), nil
-		}
-	case protoreflect.DoubleKind:
-		// Ignore 'f' for compatibility with output generated by C++,
-		// but don't remove 'f' when the value is "-inf" or "inf".
-		v := tok.value
-		if strings.HasSuffix(v, "f") && v != "-inf" && v != "inf" {
-			v = v[:len(v)-len("f")]
-		}
-		if x, err := strconv.ParseFloat(v, 64); err == nil {
-			return protoreflect.ValueOfFloat64(float64(x)), nil
-		}
-	case protoreflect.StringKind:
-		if isQuote(tok.value[0]) {
-			return protoreflect.ValueOfString(tok.unquoted), nil
-		}
-	case protoreflect.BytesKind:
-		if isQuote(tok.value[0]) {
-			return protoreflect.ValueOfBytes([]byte(tok.unquoted)), nil
-		}
-	case protoreflect.EnumKind:
-		if x, err := strconv.ParseInt(tok.value, 0, 32); err == nil {
-			return protoreflect.ValueOfEnum(protoreflect.EnumNumber(x)), nil
-		}
-		vd := fd.Enum().Values().ByName(protoreflect.Name(tok.value))
-		if vd != nil {
-			return protoreflect.ValueOfEnum(vd.Number()), nil
-		}
-	case protoreflect.MessageKind, protoreflect.GroupKind:
-		var terminator string
-		switch tok.value {
-		case "{":
-			terminator = "}"
-		case "<":
-			terminator = ">"
-		default:
-			return v, p.errorf("expected '{' or '<', found %q", tok.value)
-		}
-		err := p.unmarshalMessage(v.Message(), terminator)
-		return v, err
-	default:
-		panic(fmt.Sprintf("invalid kind %v", fd.Kind()))
-	}
-	return v, p.errorf("invalid %v: %v", fd.Kind(), tok.value)
-}
-
-// Consume a ':' from the input stream (if the next token is a colon),
-// returning an error if a colon is needed but not present.
-func (p *textParser) checkForColon(fd protoreflect.FieldDescriptor) *ParseError {
-	tok := p.next()
-	if tok.err != nil {
-		return tok.err
-	}
-	if tok.value != ":" {
-		if fd.Message() == nil {
-			return p.errorf("expected ':', found %q", tok.value)
-		}
-		p.back()
-	}
-	return nil
-}
-
-// consumeExtensionOrAnyName consumes an extension name or an Any type URL and
-// the following ']'. It returns the name or URL consumed.
-func (p *textParser) consumeExtensionOrAnyName() (string, error) {
-	tok := p.next()
-	if tok.err != nil {
-		return "", tok.err
-	}
-
-	// If extension name or type url is quoted, it's a single token.
-	if len(tok.value) > 2 && isQuote(tok.value[0]) && tok.value[len(tok.value)-1] == tok.value[0] {
-		name, err := unquoteC(tok.value[1:len(tok.value)-1], rune(tok.value[0]))
-		if err != nil {
-			return "", err
-		}
-		return name, p.consumeToken("]")
-	}
-
-	// Consume everything up to "]"
-	var parts []string
-	for tok.value != "]" {
-		parts = append(parts, tok.value)
-		tok = p.next()
-		if tok.err != nil {
-			return "", p.errorf("unrecognized type_url or extension name: %s", tok.err)
-		}
-		if p.done && tok.value != "]" {
-			return "", p.errorf("unclosed type_url or extension name")
-		}
-	}
-	return strings.Join(parts, ""), nil
-}
-
-// consumeOptionalSeparator consumes an optional semicolon or comma.
-// It is used in unmarshalMessage to provide backward compatibility.
-func (p *textParser) consumeOptionalSeparator() error {
-	tok := p.next()
-	if tok.err != nil {
-		return tok.err
-	}
-	if tok.value != ";" && tok.value != "," {
-		p.back()
-	}
-	return nil
-}
-
-func (p *textParser) errorf(format string, a ...interface{}) *ParseError {
-	pe := &ParseError{fmt.Sprintf(format, a...), p.cur.line, p.cur.offset}
-	p.cur.err = pe
-	p.done = true
-	return pe
-}
-
-func (p *textParser) skipWhitespace() {
-	i := 0
-	for i < len(p.s) && (isWhitespace(p.s[i]) || p.s[i] == '#') {
-		if p.s[i] == '#' {
-			// comment; skip to end of line or input
-			for i < len(p.s) && p.s[i] != '\n' {
-				i++
-			}
-			if i == len(p.s) {
-				break
-			}
-		}
-		if p.s[i] == '\n' {
-			p.line++
-		}
-		i++
-	}
-	p.offset += i
-	p.s = p.s[i:len(p.s)]
-	if len(p.s) == 0 {
-		p.done = true
-	}
-}
-
-func (p *textParser) advance() {
-	// Skip whitespace
-	p.skipWhitespace()
-	if p.done {
-		return
-	}
-
-	// Start of non-whitespace
-	p.cur.err = nil
-	p.cur.offset, p.cur.line = p.offset, p.line
-	p.cur.unquoted = ""
-	switch p.s[0] {
-	case '<', '>', '{', '}', ':', '[', ']', ';', ',', '/':
-		// Single symbol
-		p.cur.value, p.s = p.s[0:1], p.s[1:len(p.s)]
-	case '"', '\'':
-		// Quoted string
-		i := 1
-		for i < len(p.s) && p.s[i] != p.s[0] && p.s[i] != '\n' {
-			if p.s[i] == '\\' && i+1 < len(p.s) {
-				// skip escaped char
-				i++
-			}
-			i++
-		}
-		if i >= len(p.s) || p.s[i] != p.s[0] {
-			p.errorf("unmatched quote")
-			return
-		}
-		unq, err := unquoteC(p.s[1:i], rune(p.s[0]))
-		if err != nil {
-			p.errorf("invalid quoted string %s: %v", p.s[0:i+1], err)
-			return
-		}
-		p.cur.value, p.s = p.s[0:i+1], p.s[i+1:len(p.s)]
-		p.cur.unquoted = unq
-	default:
-		i := 0
-		for i < len(p.s) && isIdentOrNumberChar(p.s[i]) {
-			i++
-		}
-		if i == 0 {
-			p.errorf("unexpected byte %#x", p.s[0])
-			return
-		}
-		p.cur.value, p.s = p.s[0:i], p.s[i:len(p.s)]
-	}
-	p.offset += len(p.cur.value)
-}
-
-// Back off the parser by one token. Can only be done between calls to next().
-// It makes the next advance() a no-op.
-func (p *textParser) back() { p.backed = true }
-
-// Advances the parser and returns the new current token.
-func (p *textParser) next() *token {
-	if p.backed || p.done {
-		p.backed = false
-		return &p.cur
-	}
-	p.advance()
-	if p.done {
-		p.cur.value = ""
-	} else if len(p.cur.value) > 0 && isQuote(p.cur.value[0]) {
-		// Look for multiple quoted strings separated by whitespace,
-		// and concatenate them.
-		cat := p.cur
-		for {
-			p.skipWhitespace()
-			if p.done || !isQuote(p.s[0]) {
-				break
-			}
-			p.advance()
-			if p.cur.err != nil {
-				return &p.cur
-			}
-			cat.value += " " + p.cur.value
-			cat.unquoted += p.cur.unquoted
-		}
-		p.done = false // parser may have seen EOF, but we want to return cat
-		p.cur = cat
-	}
-	return &p.cur
-}
-
-func (p *textParser) consumeToken(s string) error {
-	tok := p.next()
-	if tok.err != nil {
-		return tok.err
-	}
-	if tok.value != s {
-		p.back()
-		return p.errorf("expected %q, found %q", s, tok.value)
-	}
-	return nil
-}
-
-var errBadUTF8 = errors.New("proto: bad UTF-8")
-
-func unquoteC(s string, quote rune) (string, error) {
-	// This is based on C++'s tokenizer.cc.
-	// Despite its name, this is *not* parsing C syntax.
-	// For instance, "\0" is an invalid quoted string.
-
-	// Avoid allocation in trivial cases.
-	simple := true
-	for _, r := range s {
-		if r == '\\' || r == quote {
-			simple = false
-			break
-		}
-	}
-	if simple {
-		return s, nil
-	}
-
-	buf := make([]byte, 0, 3*len(s)/2)
-	for len(s) > 0 {
-		r, n := utf8.DecodeRuneInString(s)
-		if r == utf8.RuneError && n == 1 {
-			return "", errBadUTF8
-		}
-		s = s[n:]
-		if r != '\\' {
-			if r < utf8.RuneSelf {
-				buf = append(buf, byte(r))
-			} else {
-				buf = append(buf, string(r)...)
-			}
-			continue
-		}
-
-		ch, tail, err := unescape(s)
-		if err != nil {
-			return "", err
-		}
-		buf = append(buf, ch...)
-		s = tail
-	}
-	return string(buf), nil
-}
-
-func unescape(s string) (ch string, tail string, err error) {
-	r, n := utf8.DecodeRuneInString(s)
-	if r == utf8.RuneError && n == 1 {
-		return "", "", errBadUTF8
-	}
-	s = s[n:]
-	switch r {
-	case 'a':
-		return "\a", s, nil
-	case 'b':
-		return "\b", s, nil
-	case 'f':
-		return "\f", s, nil
-	case 'n':
-		return "\n", s, nil
-	case 'r':
-		return "\r", s, nil
-	case 't':
-		return "\t", s, nil
-	case 'v':
-		return "\v", s, nil
-	case '?':
-		return "?", s, nil // trigraph workaround
-	case '\'', '"', '\\':
-		return string(r), s, nil
-	case '0', '1', '2', '3', '4', '5', '6', '7':
-		if len(s) < 2 {
-			return "", "", fmt.Errorf(`\%c requires 2 following digits`, r)
-		}
-		ss := string(r) + s[:2]
-		s = s[2:]
-		i, err := strconv.ParseUint(ss, 8, 8)
-		if err != nil {
-			return "", "", fmt.Errorf(`\%s contains non-octal digits`, ss)
-		}
-		return string([]byte{byte(i)}), s, nil
-	case 'x', 'X', 'u', 'U':
-		var n int
-		switch r {
-		case 'x', 'X':
-			n = 2
-		case 'u':
-			n = 4
-		case 'U':
-			n = 8
-		}
-		if len(s) < n {
-			return "", "", fmt.Errorf(`\%c requires %d following digits`, r, n)
-		}
-		ss := s[:n]
-		s = s[n:]
-		i, err := strconv.ParseUint(ss, 16, 64)
-		if err != nil {
-			return "", "", fmt.Errorf(`\%c%s contains non-hexadecimal digits`, r, ss)
-		}
-		if r == 'x' || r == 'X' {
-			return string([]byte{byte(i)}), s, nil
-		}
-		if i > utf8.MaxRune {
-			return "", "", fmt.Errorf(`\%c%s is not a valid Unicode code point`, r, ss)
-		}
-		return string(rune(i)), s, nil
-	}
-	return "", "", fmt.Errorf(`unknown escape \%c`, r)
-}
-
-func isIdentOrNumberChar(c byte) bool {
-	switch {
-	case 'A' <= c && c <= 'Z', 'a' <= c && c <= 'z':
-		return true
-	case '0' <= c && c <= '9':
-		return true
-	}
-	switch c {
-	case '-', '+', '.', '_':
-		return true
-	}
-	return false
-}
-
-func isWhitespace(c byte) bool {
-	switch c {
-	case ' ', '\t', '\n', '\r':
-		return true
-	}
-	return false
-}
-
-func isQuote(c byte) bool {
-	switch c {
-	case '"', '\'':
-		return true
-	}
-	return false
-}

vendor/github.com/golang/protobuf/proto/text_encode.go

@@ -1,560 +0,0 @@
-// Copyright 2010 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package proto
-
-import (
-	"bytes"
-	"encoding"
-	"fmt"
-	"io"
-	"math"
-	"sort"
-	"strings"
-
-	"google.golang.org/protobuf/encoding/prototext"
-	"google.golang.org/protobuf/encoding/protowire"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/reflect/protoreflect"
-	"google.golang.org/protobuf/reflect/protoregistry"
-)
-
-const wrapTextMarshalV2 = false
-
-// TextMarshaler is a configurable text format marshaler.
-type TextMarshaler struct {
-	Compact   bool // use compact text format (one line)
-	ExpandAny bool // expand google.protobuf.Any messages of known types
-}
-
-// Marshal writes the proto text format of m to w.
-func (tm *TextMarshaler) Marshal(w io.Writer, m Message) error {
-	b, err := tm.marshal(m)
-	if len(b) > 0 {
-		if _, err := w.Write(b); err != nil {
-			return err
-		}
-	}
-	return err
-}
-
-// Text returns a proto text formatted string of m.
-func (tm *TextMarshaler) Text(m Message) string {
-	b, _ := tm.marshal(m)
-	return string(b)
-}
-
-func (tm *TextMarshaler) marshal(m Message) ([]byte, error) {
-	mr := MessageReflect(m)
-	if mr == nil || !mr.IsValid() {
-		return []byte("<nil>"), nil
-	}
-
-	if wrapTextMarshalV2 {
-		if m, ok := m.(encoding.TextMarshaler); ok {
-			return m.MarshalText()
-		}
-
-		opts := prototext.MarshalOptions{
-			AllowPartial: true,
-			EmitUnknown:  true,
-		}
-		if !tm.Compact {
-			opts.Indent = "  "
-		}
-		if !tm.ExpandAny {
-			opts.Resolver = (*protoregistry.Types)(nil)
-		}
-		return opts.Marshal(mr.Interface())
-	} else {
-		w := &textWriter{
-			compact:   tm.Compact,
-			expandAny: tm.ExpandAny,
-			complete:  true,
-		}
-
-		if m, ok := m.(encoding.TextMarshaler); ok {
-			b, err := m.MarshalText()
-			if err != nil {
-				return nil, err
-			}
-			w.Write(b)
-			return w.buf, nil
-		}
-
-		err := w.writeMessage(mr)
-		return w.buf, err
-	}
-}
-
-var (
-	defaultTextMarshaler = TextMarshaler{}
-	compactTextMarshaler = TextMarshaler{Compact: true}
-)
-
-// MarshalText writes the proto text format of m to w.
-func MarshalText(w io.Writer, m Message) error { return defaultTextMarshaler.Marshal(w, m) }
-
-// MarshalTextString returns a proto text formatted string of m.
-func MarshalTextString(m Message) string { return defaultTextMarshaler.Text(m) }
-
-// CompactText writes the compact proto text format of m to w.
-func CompactText(w io.Writer, m Message) error { return compactTextMarshaler.Marshal(w, m) }
-
-// CompactTextString returns a compact proto text formatted string of m.
-func CompactTextString(m Message) string { return compactTextMarshaler.Text(m) }
-
-var (
-	newline         = []byte("\n")
-	endBraceNewline = []byte("}\n")
-	posInf          = []byte("inf")
-	negInf          = []byte("-inf")
-	nan             = []byte("nan")
-)
-
-// textWriter is an io.Writer that tracks its indentation level.
-type textWriter struct {
-	compact   bool // same as TextMarshaler.Compact
-	expandAny bool // same as TextMarshaler.ExpandAny
-	complete  bool // whether the current position is a complete line
-	indent    int  // indentation level; never negative
-	buf       []byte
-}
-
-func (w *textWriter) Write(p []byte) (n int, _ error) {
-	newlines := bytes.Count(p, newline)
-	if newlines == 0 {
-		if !w.compact && w.complete {
-			w.writeIndent()
-		}
-		w.buf = append(w.buf, p...)
-		w.complete = false
-		return len(p), nil
-	}
-
-	frags := bytes.SplitN(p, newline, newlines+1)
-	if w.compact {
-		for i, frag := range frags {
-			if i > 0 {
-				w.buf = append(w.buf, ' ')
-				n++
-			}
-			w.buf = append(w.buf, frag...)
-			n += len(frag)
-		}
-		return n, nil
-	}
-
-	for i, frag := range frags {
-		if w.complete {
-			w.writeIndent()
-		}
-		w.buf = append(w.buf, frag...)
-		n += len(frag)
-		if i+1 < len(frags) {
-			w.buf = append(w.buf, '\n')
-			n++
-		}
-	}
-	w.complete = len(frags[len(frags)-1]) == 0
-	return n, nil
-}
-
-func (w *textWriter) WriteByte(c byte) error {
-	if w.compact && c == '\n' {
-		c = ' '
-	}
-	if !w.compact && w.complete {
-		w.writeIndent()
-	}
-	w.buf = append(w.buf, c)
-	w.complete = c == '\n'
-	return nil
-}
-
-func (w *textWriter) writeName(fd protoreflect.FieldDescriptor) {
-	if !w.compact && w.complete {
-		w.writeIndent()
-	}
-	w.complete = false
-
-	if fd.Kind() != protoreflect.GroupKind {
-		w.buf = append(w.buf, fd.Name()...)
-		w.WriteByte(':')
-	} else {
-		// Use message type name for group field name.
-		w.buf = append(w.buf, fd.Message().Name()...)
-	}
-
-	if !w.compact {
-		w.WriteByte(' ')
-	}
-}
-
-func requiresQuotes(u string) bool {
-	// When type URL contains any characters except [0-9A-Za-z./\-]*, it must be quoted.
-	for _, ch := range u {
-		switch {
-		case ch == '.' || ch == '/' || ch == '_':
-			continue
-		case '0' <= ch && ch <= '9':
-			continue
-		case 'A' <= ch && ch <= 'Z':
-			continue
-		case 'a' <= ch && ch <= 'z':
-			continue
-		default:
-			return true
-		}
-	}
-	return false
-}
-
-// writeProto3Any writes an expanded google.protobuf.Any message.
-//
-// It returns (false, nil) if sv value can't be unmarshaled (e.g. because
-// required messages are not linked in).
-//
-// It returns (true, error) when sv was written in expanded format or an error
-// was encountered.
-func (w *textWriter) writeProto3Any(m protoreflect.Message) (bool, error) {
-	md := m.Descriptor()
-	fdURL := md.Fields().ByName("type_url")
-	fdVal := md.Fields().ByName("value")
-
-	url := m.Get(fdURL).String()
-	mt, err := protoregistry.GlobalTypes.FindMessageByURL(url)
-	if err != nil {
-		return false, nil
-	}
-
-	b := m.Get(fdVal).Bytes()
-	m2 := mt.New()
-	if err := proto.Unmarshal(b, m2.Interface()); err != nil {
-		return false, nil
-	}
-	w.Write([]byte("["))
-	if requiresQuotes(url) {
-		w.writeQuotedString(url)
-	} else {
-		w.Write([]byte(url))
-	}
-	if w.compact {
-		w.Write([]byte("]:<"))
-	} else {
-		w.Write([]byte("]: <\n"))
-		w.indent++
-	}
-	if err := w.writeMessage(m2); err != nil {
-		return true, err
-	}
-	if w.compact {
-		w.Write([]byte("> "))
-	} else {
-		w.indent--
-		w.Write([]byte(">\n"))
-	}
-	return true, nil
-}
-
-func (w *textWriter) writeMessage(m protoreflect.Message) error {
-	md := m.Descriptor()
-	if w.expandAny && md.FullName() == "google.protobuf.Any" {
-		if canExpand, err := w.writeProto3Any(m); canExpand {
-			return err
-		}
-	}
-
-	fds := md.Fields()
-	for i := 0; i < fds.Len(); {
-		fd := fds.Get(i)
-		if od := fd.ContainingOneof(); od != nil {
-			fd = m.WhichOneof(od)
-			i += od.Fields().Len()
-		} else {
-			i++
-		}
-		if fd == nil || !m.Has(fd) {
-			continue
-		}
-
-		switch {
-		case fd.IsList():
-			lv := m.Get(fd).List()
-			for j := 0; j < lv.Len(); j++ {
-				w.writeName(fd)
-				v := lv.Get(j)
-				if err := w.writeSingularValue(v, fd); err != nil {
-					return err
-				}
-				w.WriteByte('\n')
-			}
-		case fd.IsMap():
-			kfd := fd.MapKey()
-			vfd := fd.MapValue()
-			mv := m.Get(fd).Map()
-
-			type entry struct{ key, val protoreflect.Value }
-			var entries []entry
-			mv.Range(func(k protoreflect.MapKey, v protoreflect.Value) bool {
-				entries = append(entries, entry{k.Value(), v})
-				return true
-			})
-			sort.Slice(entries, func(i, j int) bool {
-				switch kfd.Kind() {
-				case protoreflect.BoolKind:
-					return !entries[i].key.Bool() && entries[j].key.Bool()
-				case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind, protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
-					return entries[i].key.Int() < entries[j].key.Int()
-				case protoreflect.Uint32Kind, protoreflect.Fixed32Kind, protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
-					return entries[i].key.Uint() < entries[j].key.Uint()
-				case protoreflect.StringKind:
-					return entries[i].key.String() < entries[j].key.String()
-				default:
-					panic("invalid kind")
-				}
-			})
-			for _, entry := range entries {
-				w.writeName(fd)
-				w.WriteByte('<')
-				if !w.compact {
-					w.WriteByte('\n')
-				}
-				w.indent++
-				w.writeName(kfd)
-				if err := w.writeSingularValue(entry.key, kfd); err != nil {
-					return err
-				}
-				w.WriteByte('\n')
-				w.writeName(vfd)
-				if err := w.writeSingularValue(entry.val, vfd); err != nil {
-					return err
-				}
-				w.WriteByte('\n')
-				w.indent--
-				w.WriteByte('>')
-				w.WriteByte('\n')
-			}
-		default:
-			w.writeName(fd)
-			if err := w.writeSingularValue(m.Get(fd), fd); err != nil {
-				return err
-			}
-			w.WriteByte('\n')
-		}
-	}
-
-	if b := m.GetUnknown(); len(b) > 0 {
-		w.writeUnknownFields(b)
-	}
-	return w.writeExtensions(m)
-}
-
-func (w *textWriter) writeSingularValue(v protoreflect.Value, fd protoreflect.FieldDescriptor) error {
-	switch fd.Kind() {
-	case protoreflect.FloatKind, protoreflect.DoubleKind:
-		switch vf := v.Float(); {
-		case math.IsInf(vf, +1):
-			w.Write(posInf)
-		case math.IsInf(vf, -1):
-			w.Write(negInf)
-		case math.IsNaN(vf):
-			w.Write(nan)
-		default:
-			fmt.Fprint(w, v.Interface())
-		}
-	case protoreflect.StringKind:
-		// NOTE: This does not validate UTF-8 for historical reasons.
-		w.writeQuotedString(string(v.String()))
-	case protoreflect.BytesKind:
-		w.writeQuotedString(string(v.Bytes()))
-	case protoreflect.MessageKind, protoreflect.GroupKind:
-		var bra, ket byte = '<', '>'
-		if fd.Kind() == protoreflect.GroupKind {
-			bra, ket = '{', '}'
-		}
-		w.WriteByte(bra)
-		if !w.compact {
-			w.WriteByte('\n')
-		}
-		w.indent++
-		m := v.Message()
-		if m2, ok := m.Interface().(encoding.TextMarshaler); ok {
-			b, err := m2.MarshalText()
-			if err != nil {
-				return err
-			}
-			w.Write(b)
-		} else {
-			w.writeMessage(m)
-		}
-		w.indent--
-		w.WriteByte(ket)
-	case protoreflect.EnumKind:
-		if ev := fd.Enum().Values().ByNumber(v.Enum()); ev != nil {
-			fmt.Fprint(w, ev.Name())
-		} else {
-			fmt.Fprint(w, v.Enum())
-		}
-	default:
-		fmt.Fprint(w, v.Interface())
-	}
-	return nil
-}
-
-// writeQuotedString writes a quoted string in the protocol buffer text format.
-func (w *textWriter) writeQuotedString(s string) {
-	w.WriteByte('"')
-	for i := 0; i < len(s); i++ {
-		switch c := s[i]; c {
-		case '\n':
-			w.buf = append(w.buf, `\n`...)
-		case '\r':
-			w.buf = append(w.buf, `\r`...)
-		case '\t':
-			w.buf = append(w.buf, `\t`...)
-		case '"':
-			w.buf = append(w.buf, `\"`...)
-		case '\\':
-			w.buf = append(w.buf, `\\`...)
-		default:
-			if isPrint := c >= 0x20 && c < 0x7f; isPrint {
-				w.buf = append(w.buf, c)
-			} else {
-				w.buf = append(w.buf, fmt.Sprintf(`\%03o`, c)...)
-			}
-		}
-	}
-	w.WriteByte('"')
-}
-
-func (w *textWriter) writeUnknownFields(b []byte) {
-	if !w.compact {
-		fmt.Fprintf(w, "/* %d unknown bytes */\n", len(b))
-	}
-
-	for len(b) > 0 {
-		num, wtyp, n := protowire.ConsumeTag(b)
-		if n < 0 {
-			return
-		}
-		b = b[n:]
-
-		if wtyp == protowire.EndGroupType {
-			w.indent--
-			w.Write(endBraceNewline)
-			continue
-		}
-		fmt.Fprint(w, num)
-		if wtyp != protowire.StartGroupType {
-			w.WriteByte(':')
-		}
-		if !w.compact || wtyp == protowire.StartGroupType {
-			w.WriteByte(' ')
-		}
-		switch wtyp {
-		case protowire.VarintType:
-			v, n := protowire.ConsumeVarint(b)
-			if n < 0 {
-				return
-			}
-			b = b[n:]
-			fmt.Fprint(w, v)
-		case protowire.Fixed32Type:
-			v, n := protowire.ConsumeFixed32(b)
-			if n < 0 {
-				return
-			}
-			b = b[n:]
-			fmt.Fprint(w, v)
-		case protowire.Fixed64Type:
-			v, n := protowire.ConsumeFixed64(b)
-			if n < 0 {
-				return
-			}
-			b = b[n:]
-			fmt.Fprint(w, v)
-		case protowire.BytesType:
-			v, n := protowire.ConsumeBytes(b)
-			if n < 0 {
-				return
-			}
-			b = b[n:]
-			fmt.Fprintf(w, "%q", v)
-		case protowire.StartGroupType:
-			w.WriteByte('{')
-			w.indent++
-		default:
-			fmt.Fprintf(w, "/* unknown wire type %d */", wtyp)
-		}
-		w.WriteByte('\n')
-	}
-}
-
-// writeExtensions writes all the extensions in m.
-func (w *textWriter) writeExtensions(m protoreflect.Message) error {
-	md := m.Descriptor()
-	if md.ExtensionRanges().Len() == 0 {
-		return nil
-	}
-
-	type ext struct {
-		desc protoreflect.FieldDescriptor
-		val  protoreflect.Value
-	}
-	var exts []ext
-	m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
-		if fd.IsExtension() {
-			exts = append(exts, ext{fd, v})
-		}
-		return true
-	})
-	sort.Slice(exts, func(i, j int) bool {
-		return exts[i].desc.Number() < exts[j].desc.Number()
-	})
-
-	for _, ext := range exts {
-		// For message set, use the name of the message as the extension name.
-		name := string(ext.desc.FullName())
-		if isMessageSet(ext.desc.ContainingMessage()) {
-			name = strings.TrimSuffix(name, ".message_set_extension")
-		}
-
-		if !ext.desc.IsList() {
-			if err := w.writeSingularExtension(name, ext.val, ext.desc); err != nil {
-				return err
-			}
-		} else {
-			lv := ext.val.List()
-			for i := 0; i < lv.Len(); i++ {
-				if err := w.writeSingularExtension(name, lv.Get(i), ext.desc); err != nil {
-					return err
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func (w *textWriter) writeSingularExtension(name string, v protoreflect.Value, fd protoreflect.FieldDescriptor) error {
-	fmt.Fprintf(w, "[%s]:", name)
-	if !w.compact {
-		w.WriteByte(' ')
-	}
-	if err := w.writeSingularValue(v, fd); err != nil {
-		return err
-	}
-	w.WriteByte('\n')
-	return nil
-}
-
-func (w *textWriter) writeIndent() {
-	if !w.complete {
-		return
-	}
-	for i := 0; i < w.indent*2; i++ {
-		w.buf = append(w.buf, ' ')
-	}
-	w.complete = false
-}

vendor/github.com/golang/protobuf/proto/wire.go

@@ -1,78 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package proto
-
-import (
-	protoV2 "google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/runtime/protoiface"
-)
-
-// Size returns the size in bytes of the wire-format encoding of m.
-func Size(m Message) int {
-	if m == nil {
-		return 0
-	}
-	mi := MessageV2(m)
-	return protoV2.Size(mi)
-}
-
-// Marshal returns the wire-format encoding of m.
-func Marshal(m Message) ([]byte, error) {
-	b, err := marshalAppend(nil, m, false)
-	if b == nil {
-		b = zeroBytes
-	}
-	return b, err
-}
-
-var zeroBytes = make([]byte, 0, 0)
-
-func marshalAppend(buf []byte, m Message, deterministic bool) ([]byte, error) {
-	if m == nil {
-		return nil, ErrNil
-	}
-	mi := MessageV2(m)
-	nbuf, err := protoV2.MarshalOptions{
-		Deterministic: deterministic,
-		AllowPartial:  true,
-	}.MarshalAppend(buf, mi)
-	if err != nil {
-		return buf, err
-	}
-	if len(buf) == len(nbuf) {
-		if !mi.ProtoReflect().IsValid() {
-			return buf, ErrNil
-		}
-	}
-	return nbuf, checkRequiredNotSet(mi)
-}
-
-// Unmarshal parses a wire-format message in b and places the decoded results in m.
-//
-// Unmarshal resets m before starting to unmarshal, so any existing data in m is always
-// removed. Use UnmarshalMerge to preserve and append to existing data.
-func Unmarshal(b []byte, m Message) error {
-	m.Reset()
-	return UnmarshalMerge(b, m)
-}
-
-// UnmarshalMerge parses a wire-format message in b and places the decoded results in m.
-func UnmarshalMerge(b []byte, m Message) error {
-	mi := MessageV2(m)
-	out, err := protoV2.UnmarshalOptions{
-		AllowPartial: true,
-		Merge:        true,
-	}.UnmarshalState(protoiface.UnmarshalInput{
-		Buf:     b,
-		Message: mi.ProtoReflect(),
-	})
-	if err != nil {
-		return err
-	}
-	if out.Flags&protoiface.UnmarshalInitialized > 0 {
-		return nil
-	}
-	return checkRequiredNotSet(mi)
-}

vendor/github.com/golang/protobuf/proto/wrappers.go

@@ -1,34 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package proto
-
-// Bool stores v in a new bool value and returns a pointer to it.
-func Bool(v bool) *bool { return &v }
-
-// Int stores v in a new int32 value and returns a pointer to it.
-//
-// Deprecated: Use Int32 instead.
-func Int(v int) *int32 { return Int32(int32(v)) }
-
-// Int32 stores v in a new int32 value and returns a pointer to it.
-func Int32(v int32) *int32 { return &v }
-
-// Int64 stores v in a new int64 value and returns a pointer to it.
-func Int64(v int64) *int64 { return &v }
-
-// Uint32 stores v in a new uint32 value and returns a pointer to it.
-func Uint32(v uint32) *uint32 { return &v }
-
-// Uint64 stores v in a new uint64 value and returns a pointer to it.
-func Uint64(v uint64) *uint64 { return &v }
-
-// Float32 stores v in a new float32 value and returns a pointer to it.
-func Float32(v float32) *float32 { return &v }
-
-// Float64 stores v in a new float64 value and returns a pointer to it.
-func Float64(v float64) *float64 { return &v }
-
-// String stores v in a new string value and returns a pointer to it.
-func String(v string) *string { return &v }