diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..f241c16 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,67 @@ +run: + tests: false + skip-dirs: + - data + - vendor + modules-download-mode: readonly + allow-parallel-runners: true + go: '1.21' +output: + sort-results: true +linters-settings: + gosec: + excludes: + # Оно не используется для повышения безопасности или каких-то таких секъюрных вещей, оно используется для выборки + # произвольного элемента из массива, так что ИДИ НАХУЙ с этой проверкой и не еби мозги липовой безопасностью. + - G404 # Insecure random number source (rand) +linters: + disable-all: true + enable: + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - unused + - asasalint + - asciicheck + - bodyclose + - containedctx + - contextcheck + - decorder + - durationcheck + - errchkjson + - errorlint + - exhaustive + - exportloopref + - gci + - ginkgolinter + - gocheckcompilerdirectives + - gocritic + - godot + - gofmt + - goimports + - gosec + - gosmopolitan + - grouper + - importas + - ireturn + - makezero + - mirror + - nakedret + - nilerr + - nilnil + - nlreturn + - noctx + - nolintlint + - nonamedreturns + - predeclared + - promlinter + - reassign + - revive + - rowserrcheck + - stylecheck + - unconvert + - wastedassign + - whitespace + - wsl diff --git a/config.go b/config.go index ce8e69e..26ecfe7 100644 --- a/config.go +++ b/config.go @@ -2,7 +2,6 @@ package main import ( "encoding/json" - "errors" "fmt" "os" @@ -21,7 +20,7 @@ func parseConfig(path string) (myConfig, error) { // Конфиг-файл длинноват для конфига, попробуем следующего кандидата. if fileInfo.Size() > 65535 { - err := errors.New(fmt.Sprintf("Config file %s is too long for config, skipping", path)) + err := fmt.Errorf("Config file %s is too long for config, skipping", path) return myConfig{}, err } @@ -36,8 +35,11 @@ func parseConfig(path string) (myConfig, error) { // Исходя из документации, hjson какбы умеет парсить "кривой" json, но парсит его в map-ку. // Интереснее на выходе получить структурку: то есть мы вначале конфиг преобразуем в map-ку, затем эту map-ку // сериализуем в json, а потом json преврщааем в стркутурку. Не очень эффективно, но он и не часто требуется. - var sampleConfig myConfig - var tmp map[string]interface{} + var ( + sampleConfig myConfig + tmp map[string]interface{} + ) + err = hjson.Unmarshal(buf, &tmp) // Не удалось распарсить. @@ -59,6 +61,7 @@ func parseConfig(path string) (myConfig, error) { // Валидируем значения из конфига. if sampleConfig.Redis.Server == "" { sampleConfig.Redis.Server = "localhost" + log.Infof("Redis server is not defined in config, using localhost") } @@ -67,19 +70,19 @@ func parseConfig(path string) (myConfig, error) { } if sampleConfig.Redis.Channel == "" { - err := errors.New(fmt.Sprintf("Channel field in config file %s must be set", path)) + err := fmt.Errorf("Channel field in config file %s must be set", path) return myConfig{}, err } if sampleConfig.Redis.MyChannel == "" { - err := errors.New(fmt.Sprintf("My_channel field in config file %s must be set", path)) + err := fmt.Errorf("My_channel field in config file %s must be set", path) return myConfig{}, err } if sampleConfig.Telegram.Token == "" { - return myConfig{}, errors.New(fmt.Sprintf("telegram.token field in config file %s must be set", path)) + return myConfig{}, fmt.Errorf("telegram.token field in config file %s must be set", path) } if sampleConfig.Loglevel == "" { @@ -89,7 +92,7 @@ func parseConfig(path string) (myConfig, error) { // sampleConfig.Log = "" if not set if sampleConfig.Csign == "" { - err := errors.New(fmt.Sprintf("Csign field in config file %s must be set", path)) + err := fmt.Errorf("Csign field in config file %s must be set", path) return myConfig{}, err } @@ -99,7 +102,7 @@ func parseConfig(path string) (myConfig, error) { } if sampleConfig.DataDir == "" { - return myConfig{}, errors.New(fmt.Sprintf("Data_dir field in config file %s must be set", path)) + return myConfig{}, fmt.Errorf("Data_dir field in config file %s must be set", path) } return sampleConfig, err diff --git a/globals.go b/globals.go index 539bef6..c3a0de0 100644 --- a/globals.go +++ b/globals.go @@ -9,13 +9,13 @@ import ( "github.com/go-redis/redis/v8" ) -// Config - это у нас глобальная штука :) +// Config - это у нас глобальная штука. var config myConfig -// To break circular message forwarding we must set some sane default, it can be overridden via config +// To break circular message forwarding we must set some sane default, it can be overridden via config. var forwardMax int64 = 5 -// Объектики клиента-редиски +// Объектики клиента-редиски. var redisClient *redis.Client var subscriber *redis.PubSub @@ -25,10 +25,10 @@ var ctx = context.Background() // Ставится в true, если мы получили сигнал на выключение var shutdown = false -// Канал, в который приходят уведомления для хэндлера сигналов от траппера сигналов +// Канал, в который приходят уведомления для хэндлера сигналов от траппера сигналов. var sigChan = make(chan os.Signal, 1) -// Мапка с открытыми дескрипторами баз с настройками +// Мапка с открытыми дескрипторами баз с настройками. var settingsDB = make(map[string]*pebble.DB) var tg echotron.API diff --git a/go.mod b/go.mod index 507b125..488652d 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module aleesa-telegram-go go 1.19 require ( - github.com/NicoNex/echotron/v3 v3.28.0 + github.com/NicoNex/echotron/v3 v3.29.0 github.com/cockroachdb/errors v1.11.1 - github.com/cockroachdb/pebble v1.0.0 + github.com/cockroachdb/pebble v1.1.0 github.com/davecgh/go-spew v1.1.1 github.com/go-redis/redis/v8 v8.11.5 github.com/hjson/hjson-go v3.3.0+incompatible @@ -18,21 +18,22 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/getsentry/sentry-go v0.26.0 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.7 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.46.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect - golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect - golang.org/x/sys v0.16.0 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect + golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/protobuf v1.32.0 // indirect ) diff --git a/go.sum b/go.sum index 7e370ff..f6fd7a5 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,22 @@ github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -github.com/NicoNex/echotron/v3 v3.28.0 h1:A5uGWlD1iQYJe41XYxmb/sAQ8ZoDSJgbB2+Ix9PDimY= -github.com/NicoNex/echotron/v3 v3.28.0/go.mod h1:LpP5IyHw0y+DZUZMBgXEDAF9O8feXrQu7w7nlJzzoZI= +github.com/NicoNex/echotron/v3 v3.29.0 h1:STzeYNauF+h82vTL9QIqIBEvhke/bsxgYKQMCmJKnLM= +github.com/NicoNex/echotron/v3 v3.29.0/go.mod h1:LpP5IyHw0y+DZUZMBgXEDAF9O8feXrQu7w7nlJzzoZI= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877 h1:1MLK4YpFtIEo3ZtMA5C795Wtv5VuUnrXX7mQG+aHg6o= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.0.0 h1:WZWlV/s78glZbY2ylUITDOWSVBD3cLjcWPLRPFbHNYg= -github.com/cockroachdb/pebble v1.0.0/go.mod h1:bynZ3gvVyhlvjLI7PT6dmZ7g76xzJ7HpxfjgkzCGz6s= +github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4= +github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -22,8 +24,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/getsentry/sentry-go v0.26.0 h1:IX3++sF6/4B5JcevhdZfdKIHfyvMmAq/UnqcyT2H6mA= -github.com/getsentry/sentry-go v0.26.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= @@ -31,13 +33,13 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/hjson/hjson-go v3.3.0+incompatible h1:Rqr+Ya+0aCJMjaE4s8E9YKvuJLuLVpEvz4ONum52vnI= github.com/hjson/hjson-go v3.3.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -53,10 +55,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= -github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -72,8 +74,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -89,8 +91,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= diff --git a/msg-parsers.go b/msg-parsers.go index 75be0c6..5cf955d 100644 --- a/msg-parsers.go +++ b/msg-parsers.go @@ -11,10 +11,10 @@ import ( // telego msg parser -// redisMsgParser парсит json-чики прилетевшие из REDIS-ки, причём, json-чики должны быть относительно валидными +// redisMsgParser парсит json-чики прилетевшие из REDIS-ки, причём, json-чики должны быть относительно валидными. func redisMsgParser(msg string) { if shutdown { - // Если мы завершаем работу программы, то нам ничего обрабатывать не надо + // Если мы завершаем работу программы, то нам ничего обрабатывать не надо. return } @@ -24,48 +24,56 @@ func redisMsgParser(msg string) { if err := json.Unmarshal([]byte(msg), &j); err != nil { log.Warnf("Unable to to parse message from redis channel: %s", err) + return } // Validate our j if exist := j.From; exist == "" { log.Warnf("Incorrect msg from redis, no from field: %s", msg) + return } if exist := j.Chatid; exist == "" { log.Warnf("Incorrect msg from redis, no chatid field: %s", msg) + return } if exist := j.Userid; exist == "" { log.Warnf("Incorrect msg from redis, no userid field: %s", msg) + return } if exist := j.Message; exist == "" { log.Warnf("Incorrect msg from redis, no message field: %s", msg) + return } if exist := j.Plugin; exist == "" { log.Warnf("Incorrect msg from redis, no plugin field: %s", msg) + return } if exist := j.Mode; exist == "" { log.Warnf("Incorrect msg from redis, no mode field: %s", msg) + return } // j.Misc.Answer может и не быть, тогда ответа на такое сообщение не будет if j.Misc.Answer == 0 { log.Debug("Field Misc->Answer = 0, skipping message") + return } // j.Misc.BotNick тоже можно не передавать, тогда будет записана пустая строка - // j.Misc.CSign если нам его не передали, возьмём значение из конфига + // j.Misc.CSign если нам его не передали, возьмём значение из конфига. (но по идее нам оно тут не нужнО.) if exist := j.Misc.Csign; exist == "" { j.Misc.Csign = config.Csign } @@ -82,6 +90,7 @@ func redisMsgParser(msg string) { // Отвалидировались, теперь вернёмся к нашим баранам. var opts *echotron.MessageOptions + chatid, err := strconv.ParseInt(j.Chatid, 10, 64) if err != nil { @@ -100,8 +109,6 @@ func redisMsgParser(msg string) { log.Errorf("Unable to send message to telegram api: %s", err) log.Errorf("Response dump: %s", spew.Sdump(resp)) } - - return } // telegramMsgParser парсит ивент, прилетевший из bot api. @@ -128,11 +135,14 @@ func telegramMsgParser(msg *echotron.Update) { // TODO: проверить, надо ли прощаться с ушедшим участником чата и попрощаться, если надо } + var rmsg rMsg + switch msg.Message.Chat.Type { case "private": // handle private messages - // всегда ставим флажок, что надо отвечать. + rmsg.Misc.Answer = 1 // если фраза является командой - засылаем её в парсер команд + // Засылаем фразу в misc-канал (в роутер) case "group", "supergroup": // handle public (super)group messages @@ -144,11 +154,14 @@ func telegramMsgParser(msg *echotron.Update) { // Здесь же, пытаемся убрать из фразы ник или имя бота // Засылаем фразу в misc-канал (в роутер) case "channel": - // Тут мы ничего сделать не можем + // Тут мы ничего сделать не можем. + } //nolint:wsl +} - } +func cmdParser(cmd string) error { + var err error - return + return err } /* vim: set ft=go noet ai ts=4 sw=4 sts=4: */ diff --git a/settings-db-util.go b/settings-db-util.go index 58763c9..e56801d 100644 --- a/settings-db-util.go +++ b/settings-db-util.go @@ -11,7 +11,7 @@ import ( log "github.com/sirupsen/logrus" ) -// StoreKV сохраняет в указанной бд ключ и значение +// StoreKV сохраняет в указанной бд ключ и значение. func StoreKV(db *pebble.DB, key string, value string) error { var kArray = []byte(key) var vArray = []byte(value) @@ -40,7 +40,7 @@ func FetchV(db *pebble.DB, key string) (string, error) { return valueString, err } -// Достанем настройку из БД с настройками +// getSetting достаёт настройку из БД с настройками. func getSetting(chatID string, setting string) string { var err error @@ -82,7 +82,7 @@ func getSetting(chatID string, setting string) string { return value } -// Сохраним настройку в БД с настройками +// saveSetting сохраняет настройку в БД с настройками func saveSetting(chatID string, setting string, value string) error { var chatHash = sha256.Sum256([]byte(chatID)) var database = fmt.Sprintf("settings_db/%x", chatHash) diff --git a/types.go b/types.go index 269391e..b118ce8 100644 --- a/types.go +++ b/types.go @@ -1,6 +1,6 @@ package main -// Конфиг +// myConfig структура, описывающая формат Конфига. type myConfig struct { Redis struct { Server string `json:"server,omitempty"` @@ -20,7 +20,7 @@ type myConfig struct { DataDir string `json:"data_dir,omitempty"` } -// Входящее сообщение из pubsub-канала redis-ки +// rMsg структура, описывающая формат входящего сообщения из pubsub-канала redis-ки. type rMsg struct { From string `json:"from,omitempty"` Chatid string `json:"chatid,omitempty"` @@ -39,7 +39,7 @@ type rMsg struct { } `json:"Misc"` } -// Исходящее сообщение в pubsub-канал redis-ки +// sMsg структура, описывающая исходящее сообщение в pubsub-канал redis-ки. type sMsg struct { From string `json:"from"` Chatid string `json:"chatid"` diff --git a/util.go b/util.go index aa3a333..a60ffc9 100644 --- a/util.go +++ b/util.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "syscall" @@ -12,6 +11,7 @@ import ( log "github.com/sirupsen/logrus" ) +// readConfig считывает и парсит конфиг-файл. func readConfig() { configLoaded := false executablePath, err := os.Executable() @@ -40,27 +40,33 @@ func readConfig() { // Конфиг-файл длинноват для конфига, попробуем следующего кандидата if fileInfo.Size() > 65535 { log.Warnf("Config file %s is too long for config, skipping", location) + continue } - buf, err := ioutil.ReadFile(location) + buf, err := os.ReadFile(location) // Не удалось прочитать, попробуем следующего кандидата if err != nil { log.Warnf("Skip reading config file %s: %s", location, err) + continue } // Исходя из документации, hjson какбы умеет парсить "кривой" json, но парсит его в map-ку. // Интереснее на выходе получить структурку: то есть мы вначале конфиг преобразуем в map-ку, затем эту map-ку // сериализуем в json, а потом json преврщааем в стркутурку. Не очень эффективно, но он и не часто требуется. - var sampleConfig myConfig - var tmp map[string]interface{} + var ( + sampleConfig myConfig + tmp map[string]interface{} + ) + err = hjson.Unmarshal(buf, &tmp) // Не удалось распарсить - попробуем следующего кандидата if err != nil { log.Warnf("Skip parsing config file %s: %s", location, err) + continue } @@ -69,17 +75,20 @@ func readConfig() { // Не удалось преобразовать map-ку в json if err != nil { log.Warnf("Skip parsing config file %s: %s", location, err) + continue } if err := json.Unmarshal(tmpjson, &sampleConfig); err != nil { log.Warnf("Skip parsing config file %s: %s", location, err) + continue } - // Валидируем значения из конфига + // Валидируем значения из конфига. if sampleConfig.Redis.Server == "" { sampleConfig.Redis.Server = "localhost" + log.Infof("Redis server is not defined in config, using localhost") } @@ -119,7 +128,9 @@ func readConfig() { config = sampleConfig configLoaded = true + log.Infof("Using %s as config file", location) + break } @@ -129,7 +140,7 @@ func readConfig() { } } -// Хэндлер сигналов закрывает все бд, все сетевые соединения и сваливает из приложения +// sigHandler хэндлер сигналов закрывает все бд, все сетевые соединения и сваливает из приложения. func sigHandler() { var err error @@ -148,10 +159,10 @@ func sigHandler() { continue } - // Чтобы не срать в логи ошибками от редиски, проставим shutdown state приложения в true + // Чтобы не срать в логи ошибками от редиски, проставим shutdown state приложения в true. shutdown = true - // Отпишемся от всех каналов и закроем коннект к редиске + // Отпишемся от всех каналов и закроем коннект к редиске. if err = subscriber.Unsubscribe(ctx); err != nil { log.Errorf("Unable to unsubscribe from redis channels cleanly: %s", err) } else { @@ -164,7 +175,7 @@ func sigHandler() { log.Debug("Close redis connection") } - // close all telego stuff + // close all telego stuff. if len(settingsDB) > 0 { log.Debug("Closing runtime irc channel settings db") diff --git a/vendor/github.com/NicoNex/echotron/v3/types.go b/vendor/github.com/NicoNex/echotron/v3/types.go index f0f75e3..f3bf191 100644 --- a/vendor/github.com/NicoNex/echotron/v3/types.go +++ b/vendor/github.com/NicoNex/echotron/v3/types.go @@ -489,11 +489,13 @@ type Chat struct { InviteLink string `json:"invite_link,omitempty"` EmojiStatusCustomEmojiID string `json:"emoji_status_custom_emoji_id,omitempty"` Type string `json:"type"` + CustomEmojiStickerSetName string `json:"custom_emoji_sticker_set_name,omitempty"` AccentColorID int `json:"accent_color_id,omitempty"` ProfileAccentColorID int `json:"profile_accent_color_id,omitempty"` EmojiStatusExpirationDate int `json:"emoji_status_expiration_date,omitempty"` MessageAutoDeleteTime int `json:"message_auto_delete_time,omitempty"` SlowModeDelay int `json:"slow_mode_delay,omitempty"` + UnrestrictBoostCount int `json:"unrestrict_boost_count,omitempty"` LinkedChatID int64 `json:"linked_chat_id,omitempty"` ID int64 `json:"id"` IsForum bool `json:"is_forum,omitempty"` @@ -553,10 +555,12 @@ type Message struct { UsersShared *UsersShared `json:"users_shared,omitempty"` ChatShared *ChatShared `json:"chat_shared,omitempty"` Story *Story `json:"story,omitempty"` + ReplyToStory *Story `json:"reply_to_story,omitempty"` ExternalReply *ExternalReplyInfo `json:"external_reply,omitempty"` Quote *TextQuote `json:"quote,omitempty"` LinkPreviewOptions *LinkPreviewOptions `json:"link_preview_options,omitempty"` ForwardOrigin *MessageOrigin `json:"forward_origin,omitempty"` + BoostAdded *ChatBoostAdded `json:"boost_added,omitempty"` MediaGroupID string `json:"media_group_id,omitempty"` ConnectedWebsite string `json:"connected_website,omitempty"` NewChatTitle string `json:"new_chat_title,omitempty"` @@ -575,6 +579,7 @@ type Message struct { Date int `json:"date"` MigrateToChatID int `json:"migrate_to_chat_id,omitempty"` EditDate int `json:"edit_date,omitempty"` + SenderBoostCount int `json:"sender_boost_count,omitempty"` DeleteChatPhoto bool `json:"delete_chat_photo,omitempty"` IsTopicMessage bool `json:"is_topic_message,omitempty"` IsAutomaticForward bool `json:"is_automatic_forward,omitempty"` @@ -1173,6 +1178,11 @@ type ChatJoinRequest struct { UserChatID int64 `json:"user_chat_id"` } +// ChatBoostAdded represents a service message about a user boosting a chat. +type ChatBoostAdded struct { + BoostCount int `json:"boost_count"` +} + // ForumTopicCreated represents a service message about a new forum topic created in the chat. type ForumTopicCreated struct { Name string `json:"name"` @@ -1238,9 +1248,11 @@ type ChatShared struct { ChatID int64 `json:"chat_id"` } -// Story represents a message about a forwarded story in the chat. -// Currently holds no information. -type Story struct{} +// Story represents a story. +type Story struct { + Chat Chat `json:"chat"` + ID int64 `json:"id"` +} type ReactionType interface { ImplementsReactionType() diff --git a/vendor/github.com/cockroachdb/pebble/.gitignore b/vendor/github.com/cockroachdb/pebble/.gitignore index 73c80d7..87ef192 100644 --- a/vendor/github.com/cockroachdb/pebble/.gitignore +++ b/vendor/github.com/cockroachdb/pebble/.gitignore @@ -1,6 +1,9 @@ +# Github action artifacts. +artifacts # Profiling artifacts. cpu.*.prof heap.prof mutex.prof +coverprofile.out # Testing artifacts meta.*.test diff --git a/vendor/github.com/cockroachdb/pebble/Makefile b/vendor/github.com/cockroachdb/pebble/Makefile index 78c9ba3..2cbf3ea 100644 --- a/vendor/github.com/cockroachdb/pebble/Makefile +++ b/vendor/github.com/cockroachdb/pebble/Makefile @@ -4,7 +4,8 @@ GOFLAGS := STRESSFLAGS := TAGS := invariants TESTS := . -LATEST_RELEASE := $(shell git fetch origin && git branch -r --list '*/crl-release-*' | grep -o 'crl-release-.*$$' | sort | tail -1) +PREV_RELEASE := crl-release-23.1 +COVER_PROFILE := coverprofile.out .PHONY: all all: @@ -15,7 +16,10 @@ all: @echo " make stressrace" @echo " make stressmeta" @echo " make crossversion-meta" + @echo " make testcoverage" @echo " make mod-update" + @echo " make generate" + @echo " make generate-test-data" @echo " make clean" override testflags := @@ -23,6 +27,10 @@ override testflags := test: ${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG} +.PHONY: testcoverage +testcoverage: + ${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG} -coverprofile ${COVER_PROFILE} + .PHONY: testrace testrace: testflags += -race -timeout 20m testrace: test @@ -34,6 +42,10 @@ testmsan: export CC=clang testmsan: testflags += -msan -timeout 20m testmsan: test +.PHONY: testobjiotracing +testobjiotracing: + ${GO} test -tags '$(TAGS) pebble_obj_io_tracing' ${testflags} -run ${TESTS} ./objstorage/objstorageprovider/objiotracing + .PHONY: lint lint: ${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ./internal/lint @@ -51,20 +63,31 @@ stressmeta: stress .PHONY: crossversion-meta crossversion-meta: - git checkout ${LATEST_RELEASE}; \ - ${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${LATEST_RELEASE}.test'; \ + git checkout ${PREV_RELEASE}; \ + ${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/${PREV_RELEASE}.test'; \ git checkout -; \ ${GO} test -c ./internal/metamorphic -o './internal/metamorphic/crossversion/head.test'; \ - ${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${LATEST_RELEASE},${LATEST_RELEASE},${LATEST_RELEASE}.test' --version 'HEAD,HEAD,./head.test' + ${GO} test -tags '$(TAGS)' ${testflags} -v -run 'TestMetaCrossVersion' ./internal/metamorphic/crossversion --version '${PREV_RELEASE},${PREV_RELEASE},${PREV_RELEASE}.test' --version 'HEAD,HEAD,./head.test' .PHONY: stress-crossversion stress-crossversion: - STRESS=1 ./scripts/run-crossversion-meta.sh crl-release-21.2 crl-release-22.1 crl-release-22.2 master + STRESS=1 ./scripts/run-crossversion-meta.sh crl-release-21.2 crl-release-22.1 crl-release-22.2 crl-release-23.1 master .PHONY: generate generate: ${GO} generate ${PKG} +generate: + +# Note that the output of generate-test-data is not deterministic. This should +# only be run manually as needed. +.PHONY: generate-test-data +generate-test-data: + ${GO} run -tags make_incorrect_manifests ./tool/make_incorrect_manifests.go + ${GO} run -tags make_test_find_db ./tool/make_test_find_db.go + ${GO} run -tags make_test_sstables ./tool/make_test_sstables.go + ${GO} run -tags make_test_remotecat ./tool/make_test_remotecat.go + mod-update: ${GO} get -u ${GO} mod tidy @@ -92,9 +115,11 @@ endif @${GO} mod tidy $(MAKE) git-clean-check +# TODO(radu): switch back to @latest once bogus doc changes are +# addressed; see https://github.com/cockroachdb/crlfmt/pull/44 .PHONY: format format: - go install github.com/cockroachdb/crlfmt@latest && crlfmt -w -tab 2 . + go install github.com/cockroachdb/crlfmt@44a36ec7 && crlfmt -w -tab 2 . .PHONY: format-check format-check: diff --git a/vendor/github.com/cockroachdb/pebble/README.md b/vendor/github.com/cockroachdb/pebble/README.md index 12ccdb2..e1aa479 100644 --- a/vendor/github.com/cockroachdb/pebble/README.md +++ b/vendor/github.com/cockroachdb/pebble/README.md @@ -1,4 +1,4 @@ -# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble) +# Pebble [![Build Status](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/cockroachdb/pebble/actions/workflows/ci.yaml) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble) [Coverage](https://storage.googleapis.com/crl-codecover-public/pebble/index.html) #### [Nightly benchmarks](https://cockroachdb.github.io/pebble/) @@ -8,9 +8,8 @@ RocksDB file formats and a few extensions such as range deletion tombstones, table-level bloom filters, and updates to the MANIFEST format. -Pebble intentionally does not aspire to include every feature in -RocksDB and is specifically targetting the use case and feature set -needed by CockroachDB: +Pebble intentionally does not aspire to include every feature in RocksDB and +specifically targets the use case and feature set needed by CockroachDB: * Block-based tables * Checkpoints @@ -68,6 +67,17 @@ Pebble offers several improvements over RocksDB: * Faster commit pipeline that achieves better concurrency. * Seamless merged iteration of indexed batches. The mutations in the batch conceptually occupy another memtable level. +* L0 sublevels and flush splitting for concurrent compactions out of L0 and + reduced read-amplification during heavy write load. +* Faster LSM edits in LSMs with large numbers of sstables through use of a + copy-on-write B-tree to hold file metadata. +* Delete-only compactions that drop whole sstables that fall within the bounds + of a range deletion. +* Block-property collectors and filters that enable iterators to skip tables, + index blocks and data blocks that are irrelevant, according to user-defined + properties over key-value pairs. +* Range keys API, allowing KV pairs defined over a range of keyspace with + user-defined semantics and interleaved during iteration. * Smaller, more approachable code base. See the [Pebble vs RocksDB: Implementation @@ -100,10 +110,68 @@ known incompatibilities. format. * Plain table format. Pebble does not support the plain table sstable format. -* SSTable format version 3 and 4. Pebble does not currently support - version 3 and version 4 format sstables. The sstable format version - is controlled by the `BlockBasedTableOptions::format_version` - option. See [#97](https://github.com/cockroachdb/pebble/issues/97). +* SSTable format version 3 and 4. Pebble does not support version 3 + and version 4 format sstables. The sstable format version is + controlled by the `BlockBasedTableOptions::format_version` option. + See [#97](https://github.com/cockroachdb/pebble/issues/97). + +## Format major versions + +Over time Pebble has introduced new physical file formats. Backwards +incompatible changes are made through the introduction of 'format major +versions'. By default, when Pebble opens a database, it defaults to +`FormatMostCompatible`. This version is bi-directionally compatible with RocksDB +6.2.1 (with the caveats described above). + +To opt into new formats, a user may set `FormatMajorVersion` on the +[`Options`](https://pkg.go.dev/github.com/cockroachdb/pebble#Options) +supplied to +[`Open`](https://pkg.go.dev/github.com/cockroachdb/pebble#Open), or +upgrade the format major version at runtime using +[`DB.RatchetFormatMajorVersion`](https://pkg.go.dev/github.com/cockroachdb/pebble#DB.RatchetFormatMajorVersion). +Format major version upgrades are permanent; There is no option to +return to an earlier format. + +The table below outlines the history of format major versions: + +| Name | Value | Migration | +|------------------------------------|-------|------------| +| FormatMostCompatible | 1 | No | +| FormatVersioned | 3 | No | +| FormatSetWithDelete | 4 | No | +| FormatBlockPropertyCollector | 5 | No | +| FormatSplitUserKeysMarked | 6 | Background | +| FormatSplitUserKeysMarkedCompacted | 7 | Blocking | +| FormatRangeKeys | 8 | No | +| FormatMinTableFormatPebblev1 | 9 | No | +| FormatPrePebblev1Marked | 10 | Background | +| FormatSSTableValueBlocks | 12 | No | +| FormatFlushableIngest | 13 | No | +| FormatPrePebblev1MarkedCompacted | 14 | Blocking | +| FormatDeleteSizedAndObsolete | 15 | No | +| FormatVirtualSSTables | 16 | No | + +Upgrading to a format major version with 'Background' in the migration +column may trigger background activity to rewrite physical file +formats, typically through compactions. Upgrading to a format major +version with 'Blocking' in the migration column will block until a +migration is complete. The database may continue to serve reads and +writes if upgrading a live database through +`RatchetFormatMajorVersion`, but the method call will not return until +the migration is complete. + +For reference, the table below lists the range of supported Pebble format major +versions for CockroachDB releases. + +| CockroachDB release | Earliest supported | Latest supported | +|---------------------|------------------------------------|---------------------------| +| 20.1 through 21.1 | FormatMostCompatible | FormatMostCompatible | +| 21.2 | FormatMostCompatible | FormatSetWithDelete | +| 21.2 | FormatMostCompatible | FormatSetWithDelete | +| 22.1 | FormatMostCompatible | FormatSplitUserKeysMarked | +| 22.2 | FormatMostCompatible | FormatPrePebblev1Marked | +| 23.1 | FormatSplitUserKeysMarkedCompacted | FormatFlushableIngest | +| 23.2 | FormatSplitUserKeysMarkedCompacted | FormatVirtualSSTables | ## Pedigree diff --git a/vendor/github.com/cockroachdb/pebble/batch.go b/vendor/github.com/cockroachdb/pebble/batch.go index 263894b..95c7e30 100644 --- a/vendor/github.com/cockroachdb/pebble/batch.go +++ b/vendor/github.com/cockroachdb/pebble/batch.go @@ -41,10 +41,10 @@ const ( var ErrNotIndexed = errors.New("pebble: batch not indexed") // ErrInvalidBatch indicates that a batch is invalid or otherwise corrupted. -var ErrInvalidBatch = errors.New("pebble: invalid batch") +var ErrInvalidBatch = base.MarkCorruptionError(errors.New("pebble: invalid batch")) // ErrBatchTooLarge indicates that a batch is invalid or otherwise corrupted. -var ErrBatchTooLarge = errors.Newf("pebble: batch too large: >= %s", humanize.Uint64(maxBatchSize)) +var ErrBatchTooLarge = base.MarkCorruptionError(errors.Newf("pebble: batch too large: >= %s", humanize.Bytes.Uint64(maxBatchSize))) // DeferredBatchOp represents a batch operation (eg. set, merge, delete) that is // being inserted into the batch. Indexing is not performed on the specified key @@ -183,6 +183,13 @@ func (d DeferredBatchOp) Finish() error { // WAL, and thus stable. New record kinds may be added, but the existing ones // will not be modified. type Batch struct { + batchInternal + applied atomic.Bool +} + +// batchInternal contains the set of fields within Batch that are non-atomic and +// capable of being reset using a *b = batchInternal{} struct copy. +type batchInternal struct { // Data is the wire format of a batch's log entry: // - 8 bytes for a sequence number of the first batch element, // or zeroes if the batch has not yet been applied, @@ -260,10 +267,12 @@ type Batch struct { // memtable. flushable *flushableBatch - // ingestedSSTBatch indicates that the batch contains one or more key kinds - // of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST - // then it will only contain key kinds of IngestSST. - ingestedSSTBatch bool + // minimumFormatMajorVersion indicates the format major version required in + // order to commit this batch. If an operation requires a particular format + // major version, it ratchets the batch's minimumFormatMajorVersion. When + // the batch is committed, this is validated against the database's current + // format major version. + minimumFormatMajorVersion FormatMajorVersion // Synchronous Apply uses the commit WaitGroup for both publishing the // seqnum and waiting for the WAL fsync (if needed). Asynchronous @@ -285,7 +294,22 @@ type Batch struct { commitStats BatchCommitStats commitErr error - applied uint32 // updated atomically + + // Position bools together to reduce the sizeof the struct. + + // ingestedSSTBatch indicates that the batch contains one or more key kinds + // of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST + // then it will only contain key kinds of IngestSST. + ingestedSSTBatch bool + + // committing is set to true when a batch begins to commit. It's used to + // ensure the batch is not mutated concurrently. It is not an atomic + // deliberately, so as to avoid the overhead on batch mutations. This is + // okay, because under correct usage this field will never be accessed + // concurrently. It's only under incorrect usage the memory accesses of this + // variable may violate memory safety. Since we don't use atomics here, + // false negatives are possible. + committing bool } // BatchCommitStats exposes stats related to committing a batch. @@ -312,7 +336,11 @@ type BatchCommitStats struct { // commitPipeline.Commit. SemaphoreWaitDuration time.Duration // WALQueueWaitDuration is the wait time for allocating memory blocks in the - // LogWriter (due to the LogWriter not writing fast enough). + // LogWriter (due to the LogWriter not writing fast enough). At the moment + // this is duration is always zero because a single WAL will allow + // allocating memory blocks up to the entire memtable size. In the future, + // we may pipeline WALs and bound the WAL queued blocks separately, so this + // field is preserved for that possibility. WALQueueWaitDuration time.Duration // MemTableWriteStallDuration is the wait caused by a write stall due to too // many memtables (due to not flushing fast enough). @@ -356,6 +384,14 @@ func newBatch(db *DB) *Batch { return b } +func newBatchWithSize(db *DB, size int) *Batch { + b := newBatch(db) + if cap(b.data) < size { + b.data = rawalloc.New(0, size) + } + return b +} + func newIndexedBatch(db *DB, comparer *Comparer) *Batch { i := indexedBatchPool.Get().(*indexedBatch) i.batch.cmp = comparer.Compare @@ -367,6 +403,14 @@ func newIndexedBatch(db *DB, comparer *Comparer) *Batch { return &i.batch } +func newIndexedBatchWithSize(db *DB, comparer *Comparer, size int) *Batch { + b := newIndexedBatch(db, comparer) + if cap(b.data) < size { + b.data = rawalloc.New(0, size) + } + return b +} + // nextSeqNum returns the batch "sequence number" that will be given to the next // key written to the batch. During iteration keys within an indexed batch are // given a sequence number consisting of their offset within the batch combined @@ -405,17 +449,21 @@ func (b *Batch) release() { } } -func (b *Batch) refreshMemTableSize() { +func (b *Batch) refreshMemTableSize() error { b.memTableSize = 0 if len(b.data) < batchHeaderLen { - return + return nil } b.countRangeDels = 0 b.countRangeKeys = 0 + b.minimumFormatMajorVersion = 0 for r := b.Reader(); ; { - kind, key, value, ok := r.Next() + kind, key, value, ok, err := r.Next() if !ok { + if err != nil { + return err + } break } switch kind { @@ -423,12 +471,23 @@ func (b *Batch) refreshMemTableSize() { b.countRangeDels++ case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: b.countRangeKeys++ + case InternalKeyKindDeleteSized: + if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete { + b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete + } case InternalKeyKindIngestSST: + if b.minimumFormatMajorVersion < FormatFlushableIngest { + b.minimumFormatMajorVersion = FormatFlushableIngest + } // This key kind doesn't contribute to the memtable size. continue } b.memTableSize += memTableEntrySize(len(key), len(value)) } + if b.countRangeKeys > 0 && b.minimumFormatMajorVersion < FormatRangeKeys { + b.minimumFormatMajorVersion = FormatRangeKeys + } + return nil } // Apply the operations contained in the batch to the receiver batch. @@ -442,7 +501,7 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error { return nil } if len(batch.data) < batchHeaderLen { - return base.CorruptionErrorf("pebble: invalid batch") + return ErrInvalidBatch } offset := len(b.data) @@ -459,8 +518,11 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error { // order to update the index. for iter := BatchReader(b.data[offset:]); len(iter) > 0; { offset := uintptr(unsafe.Pointer(&iter[0])) - uintptr(unsafe.Pointer(&b.data[0])) - kind, key, value, ok := iter.Next() + kind, key, value, ok, err := iter.Next() if !ok { + if err != nil { + return err + } break } switch kind { @@ -516,6 +578,9 @@ func (b *Batch) Get(key []byte) ([]byte, io.Closer, error) { } func (b *Batch) prepareDeferredKeyValueRecord(keyLen, valueLen int, kind InternalKeyKind) { + if b.committing { + panic("pebble: batch already committing") + } if len(b.data) == 0 { b.init(keyLen + valueLen + 2*binary.MaxVarintLen64 + batchHeaderLen) } @@ -565,6 +630,9 @@ func (b *Batch) prepareDeferredKeyValueRecord(keyLen, valueLen int, kind Interna } func (b *Batch) prepareDeferredKeyRecord(keyLen int, kind InternalKeyKind) { + if b.committing { + panic("pebble: batch already committing") + } if len(b.data) == 0 { b.init(keyLen + binary.MaxVarintLen64 + batchHeaderLen) } @@ -598,6 +666,41 @@ func (b *Batch) prepareDeferredKeyRecord(keyLen int, kind InternalKeyKind) { b.data = b.data[:pos+keyLen] } +// AddInternalKey allows the caller to add an internal key of point key kinds to +// a batch. Passing in an internal key of kind RangeKey* or RangeDelete will +// result in a panic. Note that the seqnum in the internal key is effectively +// ignored, even though the Kind is preserved. This is because the batch format +// does not allow for a per-key seqnum to be specified, only a batch-wide one. +// +// Note that non-indexed keys (IngestKeyKind{LogData,IngestSST}) are not +// supported with this method as they require specialized logic. +func (b *Batch) AddInternalKey(key *base.InternalKey, value []byte, _ *WriteOptions) error { + keyLen := len(key.UserKey) + hasValue := false + switch key.Kind() { + case InternalKeyKindRangeDelete, InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: + panic("unexpected range delete or range key kind in AddInternalKey") + case InternalKeyKindSingleDelete, InternalKeyKindDelete: + b.prepareDeferredKeyRecord(len(key.UserKey), key.Kind()) + default: + b.prepareDeferredKeyValueRecord(keyLen, len(value), key.Kind()) + hasValue = true + } + b.deferredOp.index = b.index + copy(b.deferredOp.Key, key.UserKey) + if hasValue { + copy(b.deferredOp.Value, value) + } + // TODO(peter): Manually inline DeferredBatchOp.Finish(). Mid-stack inlining + // in go1.13 will remove the need for this. + if b.index != nil { + if err := b.index.Add(b.deferredOp.offset); err != nil { + return err + } + } + return nil +} + // Set adds an action to the batch that sets the key to map to the value. // // It is safe to modify the contents of the arguments after Set returns. @@ -680,6 +783,72 @@ func (b *Batch) DeleteDeferred(keyLen int) *DeferredBatchOp { return &b.deferredOp } +// DeleteSized behaves identically to Delete, but takes an additional +// argument indicating the size of the value being deleted. DeleteSized +// should be preferred when the caller has the expectation that there exists +// a single internal KV pair for the key (eg, the key has not been +// overwritten recently), and the caller knows the size of its value. +// +// DeleteSized will record the value size within the tombstone and use it to +// inform compaction-picking heuristics which strive to reduce space +// amplification in the LSM. This "calling your shot" mechanic allows the +// storage engine to more accurately estimate and reduce space amplification. +// +// It is safe to modify the contents of the arguments after DeleteSized +// returns. +func (b *Batch) DeleteSized(key []byte, deletedValueSize uint32, _ *WriteOptions) error { + deferredOp := b.DeleteSizedDeferred(len(key), deletedValueSize) + copy(b.deferredOp.Key, key) + // TODO(peter): Manually inline DeferredBatchOp.Finish(). Check if in a + // later Go release this is unnecessary. + if b.index != nil { + if err := b.index.Add(deferredOp.offset); err != nil { + return err + } + } + return nil +} + +// DeleteSizedDeferred is similar to DeleteSized in that it adds a sized delete +// operation to the batch, except it only takes in key length instead of a +// complete key slice, letting the caller encode into the DeferredBatchOp.Key +// slice and then call Finish() on the returned object. +func (b *Batch) DeleteSizedDeferred(keyLen int, deletedValueSize uint32) *DeferredBatchOp { + if b.minimumFormatMajorVersion < FormatDeleteSizedAndObsolete { + b.minimumFormatMajorVersion = FormatDeleteSizedAndObsolete + } + + // Encode the sum of the key length and the value in the value. + v := uint64(deletedValueSize) + uint64(keyLen) + + // Encode `v` as a varint. + var buf [binary.MaxVarintLen64]byte + n := 0 + { + x := v + for x >= 0x80 { + buf[n] = byte(x) | 0x80 + x >>= 7 + n++ + } + buf[n] = byte(x) + n++ + } + + // NB: In batch entries and sstable entries, values are stored as + // varstrings. Here, the value is itself a simple varint. This results in an + // unnecessary double layer of encoding: + // varint(n) varint(deletedValueSize) + // The first varint will always be 1-byte, since a varint-encoded uint64 + // will never exceed 128 bytes. This unnecessary extra byte and wrapping is + // preserved to avoid special casing across the database, and in particular + // in sstable block decoding which is performance sensitive. + b.prepareDeferredKeyValueRecord(keyLen, n, InternalKeyKindDeleteSized) + b.deferredOp.index = b.index + copy(b.deferredOp.Value, buf[:n]) + return &b.deferredOp +} + // SingleDelete adds an action to the batch that single deletes the entry for key. // See Writer.SingleDelete for more details on the semantics of SingleDelete. // @@ -782,6 +951,9 @@ func (b *Batch) rangeKeySetDeferred(startLen, internalValueLen int) *DeferredBat func (b *Batch) incrementRangeKeysCount() { b.countRangeKeys++ + if b.minimumFormatMajorVersion < FormatRangeKeys { + b.minimumFormatMajorVersion = FormatRangeKeys + } if b.index != nil { b.rangeKeys = nil b.rangeKeysSeqNum = 0 @@ -895,6 +1067,7 @@ func (b *Batch) ingestSST(fileNum base.FileNum) { // is not reset because for the InternalKeyKindIngestSST the count is the // number of sstable paths which have been added to the batch. b.memTableSize = origMemTableSize + b.minimumFormatMajorVersion = FormatFlushableIngest } // Empty returns true if the batch is empty, and false otherwise. @@ -930,11 +1103,12 @@ func (b *Batch) SetRepr(data []byte) error { } b.data = data b.count = uint64(binary.LittleEndian.Uint32(b.countData())) + var err error if b.db != nil { // Only track memTableSize for batches that will be committed to the DB. - b.refreshMemTableSize() + err = b.refreshMemTableSize() } - return nil + return err } // NewIter returns an iterator that is unpositioned (Iterator.Valid() will @@ -944,8 +1118,8 @@ func (b *Batch) SetRepr(data []byte) error { // The returned Iterator observes all of the Batch's existing mutations, but no // later mutations. Its view can be refreshed via RefreshBatchSnapshot or // SetOptions(). -func (b *Batch) NewIter(o *IterOptions) *Iterator { - return b.NewIterWithContext(context.Background(), o) +func (b *Batch) NewIter(o *IterOptions) (*Iterator, error) { + return b.NewIterWithContext(context.Background(), o), nil } // NewIterWithContext is like NewIter, and additionally accepts a context for @@ -954,7 +1128,7 @@ func (b *Batch) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterato if b.index == nil { return &Iterator{err: ErrNotIndexed} } - return b.db.newIter(ctx, b, nil /* snapshot */, o) + return b.db.newIter(ctx, b, snapshotIterOpts{}, o) } // newInternalIter creates a new internalIterator that iterates over the @@ -1167,38 +1341,40 @@ func (b *Batch) Indexed() bool { return b.index != nil } -func (b *Batch) init(cap int) { +// init ensures that the batch data slice is initialized to meet the +// minimum required size and allocates space for the batch header. +func (b *Batch) init(size int) { n := batchInitialSize - for n < cap { + for n < size { n *= 2 } - b.data = rawalloc.New(batchHeaderLen, n) - b.setCount(0) - b.setSeqNum(0) + if cap(b.data) < n { + b.data = rawalloc.New(batchHeaderLen, n) + } b.data = b.data[:batchHeaderLen] + // Zero the sequence number in the header. + for i := 0; i < len(b.data); i++ { + b.data[i] = 0 + } } // Reset resets the batch for reuse. The underlying byte slice (that is -// returned by Repr()) is not modified. It is only necessary to call this +// returned by Repr()) may not be modified. It is only necessary to call this // method if a batch is explicitly being reused. Close automatically takes are // of releasing resources when appropriate for batches that are internally // being reused. func (b *Batch) Reset() { - b.count = 0 - b.countRangeDels = 0 - b.countRangeKeys = 0 - b.memTableSize = 0 - b.deferredOp = DeferredBatchOp{} - b.tombstones = nil - b.tombstonesSeqNum = 0 - b.rangeKeys = nil - b.rangeKeysSeqNum = 0 - b.flushable = nil - b.commit = sync.WaitGroup{} - b.fsyncWait = sync.WaitGroup{} - b.commitStats = BatchCommitStats{} - b.commitErr = nil - atomic.StoreUint32(&b.applied, 0) + // Zero out the struct, retaining only the fields necessary for manual + // reuse. + b.batchInternal = batchInternal{ + data: b.data, + cmp: b.cmp, + formatKey: b.formatKey, + abbreviatedKey: b.abbreviatedKey, + index: b.index, + db: b.db, + } + b.applied.Store(false) if b.data != nil { if cap(b.data) > batchMaxRetainedSize { // If the capacity of the buffer is larger than our maximum @@ -1209,13 +1385,14 @@ func (b *Batch) Reset() { } else { // Otherwise, reset the buffer for re-use. b.data = b.data[:batchHeaderLen] - b.setSeqNum(0) + // Zero the sequence number in the header. + for i := 0; i < len(b.data); i++ { + b.data[i] = 0 + } } } if b.index != nil { b.index.Init(&b.data, b.cmp, b.abbreviatedKey) - b.rangeDelIndex = nil - b.rangeKeyIndex = nil } } @@ -1287,6 +1464,11 @@ func (b *Batch) Reader() BatchReader { } func batchDecodeStr(data []byte) (odata []byte, s []byte, ok bool) { + // TODO(jackson): This will index out of bounds if there's no varint or an + // invalid varint (eg, a single 0xff byte). Correcting will add a bit of + // overhead. We could avoid that overhead whenever len(data) >= + // binary.MaxVarint32? + var v uint32 var n int ptr := unsafe.Pointer(&data[0]) @@ -1349,29 +1531,32 @@ func ReadBatch(repr []byte) (r BatchReader, count uint32) { return repr[batchHeaderLen:], count } -// Next returns the next entry in this batch. The final return value is false -// if the batch is corrupt. The end of batch is reached when len(r)==0. -func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, ok bool) { +// Next returns the next entry in this batch, if there is one. If the reader has +// reached the end of the batch, Next returns ok=false and a nil error. If the +// batch is corrupt and the next entry is illegible, Next returns ok=false and a +// non-nil error. +func (r *BatchReader) Next() (kind InternalKeyKind, ukey []byte, value []byte, ok bool, err error) { if len(*r) == 0 { - return 0, nil, nil, false + return 0, nil, nil, false, nil } kind = InternalKeyKind((*r)[0]) if kind > InternalKeyKindMax { - return 0, nil, nil, false + return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "invalid key kind 0x%x", (*r)[0]) } *r, ukey, ok = batchDecodeStr((*r)[1:]) if !ok { - return 0, nil, nil, false + return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "decoding user key") } switch kind { case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete, - InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: + InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete, + InternalKeyKindDeleteSized: *r, value, ok = batchDecodeStr(*r) if !ok { - return 0, nil, nil, false + return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "decoding %s value", kind) } } - return kind, ukey, value, true + return kind, ukey, value, true, nil } // Note: batchIter mirrors the implementation of flushableBatchIter. Keep the @@ -1500,7 +1685,8 @@ func (i *batchIter) value() []byte { switch InternalKeyKind(data[offset]) { case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete, - InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: + InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete, + InternalKeyKindDeleteSized: _, value, ok := batchDecodeStr(data[keyEnd:]) if !ok { return nil @@ -1572,7 +1758,7 @@ var _ flushable = (*flushableBatch)(nil) // interface. This allows the batch to act like a memtable and be placed in the // queue of flushable memtables. Note that the flushable batch takes ownership // of the batch data. -func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch { +func newFlushableBatch(batch *Batch, comparer *Comparer) (*flushableBatch, error) { b := &flushableBatch{ data: batch.data, cmp: comparer.Compare, @@ -1593,8 +1779,11 @@ func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch { var index uint32 for iter := BatchReader(b.data[batchHeaderLen:]); len(iter) > 0; index++ { offset := uintptr(unsafe.Pointer(&iter[0])) - uintptr(unsafe.Pointer(&b.data[0])) - kind, key, _, ok := iter.Next() + kind, key, _, ok, err := iter.Next() if !ok { + if err != nil { + return nil, err + } break } entry := flushableBatchEntry{ @@ -1666,7 +1855,7 @@ func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch { } fragmentRangeKeys(frag, it, len(rangeKeyOffsets)) } - return b + return b, nil } func (b *flushableBatch) setSeqNum(seqNum uint64) { @@ -1715,6 +1904,7 @@ func (b *flushableBatch) Swap(i, j int) { b.offsets[i], b.offsets[j] = b.offsets[j], b.offsets[i] } +// newIter is part of the flushable interface. func (b *flushableBatch) newIter(o *IterOptions) internalIterator { return &flushableBatchIter{ batch: b, @@ -1727,6 +1917,7 @@ func (b *flushableBatch) newIter(o *IterOptions) internalIterator { } } +// newFlushIter is part of the flushable interface. func (b *flushableBatch) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator { return &flushFlushableBatchIter{ flushableBatchIter: flushableBatchIter{ @@ -1740,6 +1931,7 @@ func (b *flushableBatch) newFlushIter(o *IterOptions, bytesFlushed *uint64) inte } } +// newRangeDelIter is part of the flushable interface. func (b *flushableBatch) newRangeDelIter(o *IterOptions) keyspan.FragmentIterator { if len(b.tombstones) == 0 { return nil @@ -1747,6 +1939,7 @@ func (b *flushableBatch) newRangeDelIter(o *IterOptions) keyspan.FragmentIterato return keyspan.NewIter(b.cmp, b.tombstones) } +// newRangeKeyIter is part of the flushable interface. func (b *flushableBatch) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator { if len(b.rangeKeys) == 0 { return nil @@ -1754,17 +1947,23 @@ func (b *flushableBatch) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterato return keyspan.NewIter(b.cmp, b.rangeKeys) } +// containsRangeKeys is part of the flushable interface. func (b *flushableBatch) containsRangeKeys() bool { return len(b.rangeKeys) > 0 } +// inuseBytes is part of the flushable interface. func (b *flushableBatch) inuseBytes() uint64 { return uint64(len(b.data) - batchHeaderLen) } +// totalBytes is part of the flushable interface. func (b *flushableBatch) totalBytes() uint64 { return uint64(cap(b.data)) } +// readyForFlush is part of the flushable interface. func (b *flushableBatch) readyForFlush() bool { + // A flushable batch is always ready for flush; it must be flushed together + // with the previous memtable. return true } @@ -1944,7 +2143,8 @@ func (i *flushableBatchIter) value() base.LazyValue { var ok bool switch kind { case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindRangeDelete, - InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: + InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete, + InternalKeyKindDeleteSized: keyEnd := i.offsets[i.index].keyEnd _, value, ok = batchDecodeStr(i.data[keyEnd:]) if !ok { @@ -2081,7 +2281,10 @@ func batchSort( rangeKeyIter := b.newRangeKeyIter(nil, math.MaxUint64) return pointIter, rangeDelIter, rangeKeyIter } - f := newFlushableBatch(b, b.db.opts.Comparer) + f, err := newFlushableBatch(b, b.db.opts.Comparer) + if err != nil { + panic(err) + } return f.newIter(nil), f.newRangeDelIter(nil), f.newRangeKeyIter(nil) } diff --git a/vendor/github.com/cockroachdb/pebble/checkpoint.go b/vendor/github.com/cockroachdb/pebble/checkpoint.go index cf019ad..ad70a35 100644 --- a/vendor/github.com/cockroachdb/pebble/checkpoint.go +++ b/vendor/github.com/cockroachdb/pebble/checkpoint.go @@ -188,7 +188,10 @@ func (d *DB) Checkpoint( manifestFileNum := d.mu.versions.manifestFileNum manifestSize := d.mu.versions.manifest.Size() optionsFileNum := d.optionsFileNum - + virtualBackingFiles := make(map[base.DiskFileNum]struct{}) + for diskFileNum := range d.mu.versions.backingState.fileBackingMap { + virtualBackingFiles[diskFileNum] = struct{}{} + } // Release the manifest and DB.mu so we don't block other operations on // the database. d.mu.versions.logUnlock() @@ -209,11 +212,7 @@ func (d *DB) Checkpoint( } if ckErr != nil { // Attempt to cleanup on error. - paths, _ := fs.List(destDir) - for _, path := range paths { - _ = fs.Remove(path) - } - _ = fs.Remove(destDir) + _ = fs.RemoveAll(destDir) } }() dir, ckErr = mkdirAllAndSyncParents(fs, destDir) @@ -254,7 +253,9 @@ func (d *DB) Checkpoint( } var excludedFiles map[deletedFileEntry]*fileMetadata - + // Set of FileBacking.DiskFileNum which will be required by virtual sstables + // in the checkpoint. + requiredVirtualBackingFiles := make(map[base.DiskFileNum]struct{}) // Link or copy the sstables. for l := range current.Levels { iter := current.Levels[l].Iter() @@ -270,7 +271,15 @@ func (d *DB) Checkpoint( continue } - srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, f.FileNum) + fileBacking := f.FileBacking + if f.Virtual { + if _, ok := requiredVirtualBackingFiles[fileBacking.DiskFileNum]; ok { + continue + } + requiredVirtualBackingFiles[fileBacking.DiskFileNum] = struct{}{} + } + + srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, fileBacking.DiskFileNum) destPath := fs.PathJoin(destDir, fs.PathBase(srcPath)) ckErr = vfs.LinkOrCopy(fs, srcPath, destPath) if ckErr != nil { @@ -279,7 +288,19 @@ func (d *DB) Checkpoint( } } - ckErr = d.writeCheckpointManifest(fs, formatVers, destDir, dir, manifestFileNum, manifestSize, excludedFiles) + var removeBackingTables []base.DiskFileNum + for diskFileNum := range virtualBackingFiles { + if _, ok := requiredVirtualBackingFiles[diskFileNum]; !ok { + // The backing sstable associated with fileNum is no longer + // required. + removeBackingTables = append(removeBackingTables, diskFileNum) + } + } + + ckErr = d.writeCheckpointManifest( + fs, formatVers, destDir, dir, manifestFileNum.DiskFileNum(), manifestSize, + excludedFiles, removeBackingTables, + ) if ckErr != nil { return ckErr } @@ -292,7 +313,7 @@ func (d *DB) Checkpoint( if logNum == 0 { continue } - srcPath := base.MakeFilepath(fs, d.walDirname, fileTypeLog, logNum) + srcPath := base.MakeFilepath(fs, d.walDirname, fileTypeLog, logNum.DiskFileNum()) destPath := fs.PathJoin(destDir, fs.PathBase(srcPath)) ckErr = vfs.Copy(fs, srcPath, destPath) if ckErr != nil { @@ -315,9 +336,10 @@ func (d *DB) writeCheckpointManifest( formatVers FormatMajorVersion, destDirPath string, destDir vfs.File, - manifestFileNum FileNum, + manifestFileNum base.DiskFileNum, manifestSize int64, excludedFiles map[deletedFileEntry]*fileMetadata, + removeBackingTables []base.DiskFileNum, ) error { // Copy the MANIFEST, and create a pointer to it. We copy rather // than link because additional version edits added to the @@ -346,7 +368,7 @@ func (d *DB) writeCheckpointManifest( // need to append another record with the excluded files (we cannot simply // append a record after a raw data copy; see // https://github.com/cockroachdb/cockroach/issues/100935). - r := record.NewReader(&io.LimitedReader{R: src, N: manifestSize}, manifestFileNum) + r := record.NewReader(&io.LimitedReader{R: src, N: manifestSize}, manifestFileNum.FileNum()) w := record.NewWriter(dst) for { rr, err := r.Next() @@ -369,8 +391,10 @@ func (d *DB) writeCheckpointManifest( if len(excludedFiles) > 0 { // Write out an additional VersionEdit that deletes the excluded SST files. ve := versionEdit{ - DeletedFiles: excludedFiles, + DeletedFiles: excludedFiles, + RemovedBackingTables: removeBackingTables, } + rw, err := w.Next() if err != nil { return err @@ -397,7 +421,7 @@ func (d *DB) writeCheckpointManifest( if err != nil { return err } - if err := setCurrentFunc(formatVers, manifestMarker, fs, destDirPath, destDir)(manifestFileNum); err != nil { + if err := setCurrentFunc(formatVers, manifestMarker, fs, destDirPath, destDir)(manifestFileNum.FileNum()); err != nil { return err } return manifestMarker.Close() diff --git a/vendor/github.com/cockroachdb/pebble/cleaner.go b/vendor/github.com/cockroachdb/pebble/cleaner.go index 384bff5..f9fa43b 100644 --- a/vendor/github.com/cockroachdb/pebble/cleaner.go +++ b/vendor/github.com/cockroachdb/pebble/cleaner.go @@ -4,7 +4,18 @@ package pebble -import "github.com/cockroachdb/pebble/internal/base" +import ( + "context" + "runtime/pprof" + "sync" + "time" + + "github.com/cockroachdb/errors/oserror" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/tokenbucket" +) // Cleaner exports the base.Cleaner type. type Cleaner = base.Cleaner @@ -14,3 +25,271 @@ type DeleteCleaner = base.DeleteCleaner // ArchiveCleaner exports the base.ArchiveCleaner type. type ArchiveCleaner = base.ArchiveCleaner + +type cleanupManager struct { + opts *Options + objProvider objstorage.Provider + onTableDeleteFn func(fileSize uint64) + deletePacer *deletionPacer + + // jobsCh is used as the cleanup job queue. + jobsCh chan *cleanupJob + // waitGroup is used to wait for the background goroutine to exit. + waitGroup sync.WaitGroup + + mu struct { + sync.Mutex + // totalJobs is the total number of enqueued jobs (completed or in progress). + totalJobs int + completedJobs int + completedJobsCond sync.Cond + jobsQueueWarningIssued bool + } +} + +// We can queue this many jobs before we have to block EnqueueJob. +const jobsQueueDepth = 1000 + +// obsoleteFile holds information about a file that needs to be deleted soon. +type obsoleteFile struct { + dir string + fileNum base.DiskFileNum + fileType fileType + fileSize uint64 +} + +type cleanupJob struct { + jobID int + obsoleteFiles []obsoleteFile +} + +// openCleanupManager creates a cleanupManager and starts its background goroutine. +// The cleanupManager must be Close()d. +func openCleanupManager( + opts *Options, + objProvider objstorage.Provider, + onTableDeleteFn func(fileSize uint64), + getDeletePacerInfo func() deletionPacerInfo, +) *cleanupManager { + cm := &cleanupManager{ + opts: opts, + objProvider: objProvider, + onTableDeleteFn: onTableDeleteFn, + deletePacer: newDeletionPacer(time.Now(), int64(opts.TargetByteDeletionRate), getDeletePacerInfo), + jobsCh: make(chan *cleanupJob, jobsQueueDepth), + } + cm.mu.completedJobsCond.L = &cm.mu.Mutex + cm.waitGroup.Add(1) + + go func() { + pprof.Do(context.Background(), gcLabels, func(context.Context) { + cm.mainLoop() + }) + }() + + return cm +} + +// Close stops the background goroutine, waiting until all queued jobs are completed. +// Delete pacing is disabled for the remaining jobs. +func (cm *cleanupManager) Close() { + close(cm.jobsCh) + cm.waitGroup.Wait() +} + +// EnqueueJob adds a cleanup job to the manager's queue. +func (cm *cleanupManager) EnqueueJob(jobID int, obsoleteFiles []obsoleteFile) { + job := &cleanupJob{ + jobID: jobID, + obsoleteFiles: obsoleteFiles, + } + + // Report deleted bytes to the pacer, which can use this data to potentially + // increase the deletion rate to keep up. We want to do this at enqueue time + // rather than when we get to the job, otherwise the reported bytes will be + // subject to the throttling rate which defeats the purpose. + var pacingBytes uint64 + for _, of := range obsoleteFiles { + if cm.needsPacing(of.fileType, of.fileNum) { + pacingBytes += of.fileSize + } + } + if pacingBytes > 0 { + cm.deletePacer.ReportDeletion(time.Now(), pacingBytes) + } + + cm.mu.Lock() + cm.mu.totalJobs++ + cm.maybeLogLocked() + cm.mu.Unlock() + + if invariants.Enabled && len(cm.jobsCh) >= cap(cm.jobsCh)-2 { + panic("cleanup jobs queue full") + } + + cm.jobsCh <- job +} + +// Wait until the completion of all jobs that were already queued. +// +// Does not wait for jobs that are enqueued during the call. +// +// Note that DB.mu should not be held while calling this method; the background +// goroutine needs to acquire DB.mu to update deleted table metrics. +func (cm *cleanupManager) Wait() { + cm.mu.Lock() + defer cm.mu.Unlock() + n := cm.mu.totalJobs + for cm.mu.completedJobs < n { + cm.mu.completedJobsCond.Wait() + } +} + +// mainLoop runs the manager's background goroutine. +func (cm *cleanupManager) mainLoop() { + defer cm.waitGroup.Done() + + var tb tokenbucket.TokenBucket + // Use a token bucket with 1 token / second refill rate and 1 token burst. + tb.Init(1.0, 1.0) + for job := range cm.jobsCh { + for _, of := range job.obsoleteFiles { + if of.fileType != fileTypeTable { + path := base.MakeFilepath(cm.opts.FS, of.dir, of.fileType, of.fileNum) + cm.deleteObsoleteFile(of.fileType, job.jobID, path, of.fileNum, of.fileSize) + } else { + cm.maybePace(&tb, of.fileType, of.fileNum, of.fileSize) + cm.onTableDeleteFn(of.fileSize) + cm.deleteObsoleteObject(fileTypeTable, job.jobID, of.fileNum) + } + } + cm.mu.Lock() + cm.mu.completedJobs++ + cm.mu.completedJobsCond.Broadcast() + cm.maybeLogLocked() + cm.mu.Unlock() + } +} + +func (cm *cleanupManager) needsPacing(fileType base.FileType, fileNum base.DiskFileNum) bool { + if fileType != fileTypeTable { + return false + } + meta, err := cm.objProvider.Lookup(fileType, fileNum) + if err != nil { + // The object was already removed from the provider; we won't actually + // delete anything, so we don't need to pace. + return false + } + // Don't throttle deletion of remote objects. + return !meta.IsRemote() +} + +// maybePace sleeps before deleting an object if appropriate. It is always +// called from the background goroutine. +func (cm *cleanupManager) maybePace( + tb *tokenbucket.TokenBucket, fileType base.FileType, fileNum base.DiskFileNum, fileSize uint64, +) { + if !cm.needsPacing(fileType, fileNum) { + return + } + + tokens := cm.deletePacer.PacingDelay(time.Now(), fileSize) + if tokens == 0.0 { + // The token bucket might be in debt; it could make us wait even for 0 + // tokens. We don't want that if the pacer decided throttling should be + // disabled. + return + } + // Wait for tokens. We use a token bucket instead of sleeping outright because + // the token bucket accumulates up to one second of unused tokens. + for { + ok, d := tb.TryToFulfill(tokenbucket.Tokens(tokens)) + if ok { + break + } + time.Sleep(d) + } +} + +// deleteObsoleteFile deletes a (non-object) file that is no longer needed. +func (cm *cleanupManager) deleteObsoleteFile( + fileType fileType, jobID int, path string, fileNum base.DiskFileNum, fileSize uint64, +) { + // TODO(peter): need to handle this error, probably by re-adding the + // file that couldn't be deleted to one of the obsolete slices map. + err := cm.opts.Cleaner.Clean(cm.opts.FS, fileType, path) + if oserror.IsNotExist(err) { + return + } + + switch fileType { + case fileTypeLog: + cm.opts.EventListener.WALDeleted(WALDeleteInfo{ + JobID: jobID, + Path: path, + FileNum: fileNum.FileNum(), + Err: err, + }) + case fileTypeManifest: + cm.opts.EventListener.ManifestDeleted(ManifestDeleteInfo{ + JobID: jobID, + Path: path, + FileNum: fileNum.FileNum(), + Err: err, + }) + case fileTypeTable: + panic("invalid deletion of object file") + } +} + +func (cm *cleanupManager) deleteObsoleteObject( + fileType fileType, jobID int, fileNum base.DiskFileNum, +) { + if fileType != fileTypeTable { + panic("not an object") + } + + var path string + meta, err := cm.objProvider.Lookup(fileType, fileNum) + if err != nil { + path = "" + } else { + path = cm.objProvider.Path(meta) + err = cm.objProvider.Remove(fileType, fileNum) + } + if cm.objProvider.IsNotExistError(err) { + return + } + + switch fileType { + case fileTypeTable: + cm.opts.EventListener.TableDeleted(TableDeleteInfo{ + JobID: jobID, + Path: path, + FileNum: fileNum.FileNum(), + Err: err, + }) + } +} + +// maybeLogLocked issues a log if the job queue gets 75% full and issues a log +// when the job queue gets back to less than 10% full. +// +// Must be called with cm.mu locked. +func (cm *cleanupManager) maybeLogLocked() { + const highThreshold = jobsQueueDepth * 3 / 4 + const lowThreshold = jobsQueueDepth / 10 + + jobsInQueue := cm.mu.totalJobs - cm.mu.completedJobs + + if !cm.mu.jobsQueueWarningIssued && jobsInQueue > highThreshold { + cm.mu.jobsQueueWarningIssued = true + cm.opts.Logger.Infof("cleanup falling behind; job queue has over %d jobs", highThreshold) + } + + if cm.mu.jobsQueueWarningIssued && jobsInQueue < lowThreshold { + cm.mu.jobsQueueWarningIssued = false + cm.opts.Logger.Infof("cleanup back to normal; job queue has under %d jobs", lowThreshold) + } +} diff --git a/vendor/github.com/cockroachdb/pebble/commit.go b/vendor/github.com/cockroachdb/pebble/commit.go index 38ede48..38cdbb8 100644 --- a/vendor/github.com/cockroachdb/pebble/commit.go +++ b/vendor/github.com/cockroachdb/pebble/commit.go @@ -9,7 +9,6 @@ import ( "sync" "sync/atomic" "time" - "unsafe" "github.com/cockroachdb/pebble/record" ) @@ -33,13 +32,13 @@ type commitQueue struct { // // The head index is stored in the most-significant bits so that we can // atomically add to it and the overflow is harmless. - headTail uint64 + headTail atomic.Uint64 // slots is a ring buffer of values stored in this queue. The size must be a // power of 2. A slot is in use until *both* the tail index has moved beyond // it and the slot value has been set to nil. The slot value is set to nil // atomically by the consumer and read atomically by the producer. - slots [record.SyncConcurrency]unsafe.Pointer + slots [record.SyncConcurrency]atomic.Pointer[Batch] } const dequeueBits = 32 @@ -58,7 +57,7 @@ func (q *commitQueue) pack(head, tail uint32) uint64 { } func (q *commitQueue) enqueue(b *Batch) { - ptrs := atomic.LoadUint64(&q.headTail) + ptrs := q.headTail.Load() head, tail := q.unpack(ptrs) if (tail+uint32(len(q.slots)))&(1< 2 { + info.Annotations = append(info.Annotations, "multilevel") + } return info } -func newCompaction(pc *pickedCompaction, opts *Options) *compaction { +func newCompaction( + pc *pickedCompaction, opts *Options, beganAt time.Time, provider objstorage.Provider, +) *compaction { c := &compaction{ kind: compactionKindDefault, cmp: pc.cmp, equal: opts.equal(), comparer: opts.Comparer, formatKey: opts.Comparer.FormatKey, - score: pc.score, inputs: pc.inputs, smallest: pc.smallest, largest: pc.largest, logger: opts.Logger, version: pc.version, + beganAt: beganAt, maxOutputFileSize: pc.maxOutputFileSize, maxOverlapBytes: pc.maxOverlapBytes, - l0SublevelInfo: pc.l0SublevelInfo, + pickerMetrics: pc.pickerMetrics, } c.startLevel = &c.inputs[0] + if pc.startLevel.l0SublevelInfo != nil { + c.startLevel.l0SublevelInfo = pc.startLevel.l0SublevelInfo + } c.outputLevel = &c.inputs[1] if len(pc.extraLevels) > 0 { @@ -714,20 +754,48 @@ func newCompaction(pc *pickedCompaction, opts *Options) *compaction { c.smallest.UserKey, c.largest.UserKey, c.largest.IsExclusiveSentinel()) } c.setupInuseKeyRanges() - c.kind = pc.kind + if c.kind == compactionKindDefault && c.outputLevel.files.Empty() && !c.hasExtraLevelData() && c.startLevel.files.Len() == 1 && c.grandparents.SizeSum() <= c.maxOverlapBytes { - // This compaction can be converted into a trivial move from one level + // This compaction can be converted into a move or copy from one level // to the next. We avoid such a move if there is lots of overlapping // grandparent data. Otherwise, the move could create a parent file // that will require a very expensive merge later on. - c.kind = compactionKindMove + iter := c.startLevel.files.Iter() + meta := iter.First() + isRemote := false + // We should always be passed a provider, except in some unit tests. + if provider != nil { + objMeta, err := provider.Lookup(fileTypeTable, meta.FileBacking.DiskFileNum) + if err != nil { + panic(errors.Wrapf(err, "cannot lookup table %s in provider", meta.FileBacking.DiskFileNum)) + } + isRemote = objMeta.IsRemote() + } + // Avoid a trivial move or copy if all of these are true, as rewriting a + // new file is better: + // + // 1) The source file is a virtual sstable + // 2) The existing file `meta` is on non-remote storage + // 3) The output level prefers shared storage + mustCopy := !isRemote && remote.ShouldCreateShared(opts.Experimental.CreateOnShared, c.outputLevel.level) + if mustCopy { + // If the source is virtual, it's best to just rewrite the file as all + // conditions in the above comment are met. + if !meta.Virtual { + c.kind = compactionKindCopy + } + } else { + c.kind = compactionKindMove + } } return c } -func newDeleteOnlyCompaction(opts *Options, cur *version, inputs []compactionLevel) *compaction { +func newDeleteOnlyCompaction( + opts *Options, cur *version, inputs []compactionLevel, beganAt time.Time, +) *compaction { c := &compaction{ kind: compactionKindDeleteOnly, cmp: opts.Comparer.Compare, @@ -736,6 +804,7 @@ func newDeleteOnlyCompaction(opts *Options, cur *version, inputs []compactionLev formatKey: opts.Comparer.FormatKey, logger: opts.Logger, version: cur, + beganAt: beganAt, inputs: inputs, } @@ -815,7 +884,9 @@ func adjustGrandparentOverlapBytesForFlush(c *compaction, flushingBytes uint64) } } -func newFlush(opts *Options, cur *version, baseLevel int, flushing flushableList) *compaction { +func newFlush( + opts *Options, cur *version, baseLevel int, flushing flushableList, beganAt time.Time, +) *compaction { c := &compaction{ kind: compactionKindFlush, cmp: opts.Comparer.Compare, @@ -824,6 +895,7 @@ func newFlush(opts *Options, cur *version, baseLevel int, flushing flushableList formatKey: opts.Comparer.FormatKey, logger: opts.Logger, version: cur, + beganAt: beganAt, inputs: []compactionLevel{{level: -1}, {level: 0}}, maxOutputFileSize: math.MaxUint64, maxOverlapBytes: math.MaxUint64, @@ -1210,92 +1282,69 @@ func (c *compaction) elideRangeKey(start, end []byte) bool { func (c *compaction) newInputIter( newIters tableNewIters, newRangeKeyIter keyspan.TableNewSpanIter, snapshots []uint64, ) (_ internalIterator, retErr error) { - var rangeDelIters []keyspan.FragmentIterator - var rangeKeyIters []keyspan.FragmentIterator - - if len(c.flushing) != 0 { - if len(c.flushing) == 1 { - f := c.flushing[0] - iter := f.newFlushIter(nil, &c.bytesIterated) - if rangeDelIter := f.newRangeDelIter(nil); rangeDelIter != nil { - c.rangeDelIter.Init(c.cmp, rangeDelIter) - iter = newMergingIter(c.logger, &c.stats, c.cmp, nil, iter, &c.rangeDelIter) - } - if rangeKeyIter := f.newRangeKeyIter(nil); rangeKeyIter != nil { - mi := &keyspan.MergingIter{} - mi.Init(c.cmp, rangeKeyCompactionTransform(c.equal, snapshots, c.elideRangeKey), new(keyspan.MergingBuffers), rangeKeyIter) - c.rangeKeyInterleaving.Init(c.comparer, iter, mi, nil /* hooks */, nil /* lowerBound */, nil /* upperBound */) - iter = &c.rangeKeyInterleaving - } - return iter, nil - } - iters := make([]internalIterator, 0, len(c.flushing)+1) - rangeDelIters = make([]keyspan.FragmentIterator, 0, len(c.flushing)) - rangeKeyIters = make([]keyspan.FragmentIterator, 0, len(c.flushing)) - for i := range c.flushing { - f := c.flushing[i] - iters = append(iters, f.newFlushIter(nil, &c.bytesIterated)) - rangeDelIter := f.newRangeDelIter(nil) - if rangeDelIter != nil { - rangeDelIters = append(rangeDelIters, rangeDelIter) - } - if rangeKeyIter := f.newRangeKeyIter(nil); rangeKeyIter != nil { - rangeKeyIters = append(rangeKeyIters, rangeKeyIter) + // Validate the ordering of compaction input files for defense in depth. + // TODO(jackson): Some of the CheckOrdering calls may be adapted to pass + // ProhibitSplitUserKeys if we thread the active format major version in. Or + // if we remove support for earlier FMVs, we can remove the parameter + // altogether. + if len(c.flushing) == 0 { + if c.startLevel.level >= 0 { + err := manifest.CheckOrdering(c.cmp, c.formatKey, + manifest.Level(c.startLevel.level), c.startLevel.files.Iter(), + manifest.AllowSplitUserKeys) + if err != nil { + return nil, err } } - if len(rangeDelIters) > 0 { - c.rangeDelIter.Init(c.cmp, rangeDelIters...) - iters = append(iters, &c.rangeDelIter) - } - var iter internalIterator = newMergingIter(c.logger, &c.stats, c.cmp, nil, iters...) - if len(rangeKeyIters) > 0 { - mi := &keyspan.MergingIter{} - mi.Init(c.cmp, rangeKeyCompactionTransform(c.equal, snapshots, c.elideRangeKey), new(keyspan.MergingBuffers), rangeKeyIters...) - c.rangeKeyInterleaving.Init(c.comparer, iter, mi, nil /* hooks */, nil /* lowerBound */, nil /* upperBound */) - iter = &c.rangeKeyInterleaving - } - return iter, nil - } - - if c.startLevel.level >= 0 { err := manifest.CheckOrdering(c.cmp, c.formatKey, - manifest.Level(c.startLevel.level), c.startLevel.files.Iter()) + manifest.Level(c.outputLevel.level), c.outputLevel.files.Iter(), + manifest.AllowSplitUserKeys) if err != nil { return nil, err } - } - err := manifest.CheckOrdering(c.cmp, c.formatKey, - manifest.Level(c.outputLevel.level), c.outputLevel.files.Iter()) - if err != nil { - return nil, err - } - - if c.startLevel.level == 0 { - if c.l0SublevelInfo == nil { - panic("l0SublevelInfo not created for compaction out of L0") + if c.startLevel.level == 0 { + if c.startLevel.l0SublevelInfo == nil { + panic("l0SublevelInfo not created for compaction out of L0") + } + for _, info := range c.startLevel.l0SublevelInfo { + err := manifest.CheckOrdering(c.cmp, c.formatKey, + info.sublevel, info.Iter(), + // NB: L0 sublevels have never allowed split user keys. + manifest.ProhibitSplitUserKeys) + if err != nil { + return nil, err + } + } } - - for _, info := range c.l0SublevelInfo { + if len(c.extraLevels) > 0 { + if len(c.extraLevels) > 1 { + panic("n>2 multi level compaction not implemented yet") + } + interLevel := c.extraLevels[0] err := manifest.CheckOrdering(c.cmp, c.formatKey, - info.sublevel, info.Iter()) + manifest.Level(interLevel.level), interLevel.files.Iter(), + manifest.AllowSplitUserKeys) if err != nil { return nil, err } } } - if len(c.extraLevels) > 0 { - if len(c.extraLevels) > 1 { - panic("n>2 multi level compaction not implemented yet") - } - interLevel := c.extraLevels[0] - err := manifest.CheckOrdering(c.cmp, c.formatKey, - manifest.Level(interLevel.level), interLevel.files.Iter()) - if err != nil { - return nil, err - } - } - iters := make([]internalIterator, 0, len(c.inputs)*c.startLevel.files.Len()+1) + // There are three classes of keys that a compaction needs to process: point + // keys, range deletion tombstones and range keys. Collect all iterators for + // all these classes of keys from all the levels. We'll aggregate them + // together farther below. + // + // numInputLevels is an approximation of the number of iterator levels. Due + // to idiosyncrasies in iterator construction, we may (rarely) exceed this + // initial capacity. + numInputLevels := max[int](len(c.flushing), len(c.inputs)) + iters := make([]internalIterator, 0, numInputLevels) + rangeDelIters := make([]keyspan.FragmentIterator, 0, numInputLevels) + rangeKeyIters := make([]keyspan.FragmentIterator, 0, numInputLevels) + + // If construction of the iterator inputs fails, ensure that we close all + // the consitutent iterators. defer func() { if retErr != nil { for _, iter := range iters { @@ -1308,161 +1357,103 @@ func (c *compaction) newInputIter( } } }() - - // In normal operation, levelIter iterates over the point operations in a - // level, and initializes a rangeDelIter pointer for the range deletions in - // each table. During compaction, we want to iterate over the merged view of - // point operations and range deletions. In order to do this we create one - // levelIter per level to iterate over the point operations, and collect up - // all the range deletion files. - // - // The range deletion levels are first combined with a keyspan.MergingIter - // (currently wrapped by a keyspan.InternalIteratorShim to satisfy the - // internal iterator interface). The resulting merged rangedel iterator is - // then included with the point levels in a single mergingIter. - newRangeDelIter := func( - f manifest.LevelFile, _ *IterOptions, bytesIterated *uint64, - ) (keyspan.FragmentIterator, error) { - iter, rangeDelIter, err := newIters(context.Background(), f.FileMetadata, - nil /* iter options */, internalIterOpts{bytesIterated: &c.bytesIterated}) - if err == nil { - // TODO(peter): It is mildly wasteful to open the point iterator only to - // immediately close it. One way to solve this would be to add new - // methods to tableCache for creating point and range-deletion iterators - // independently. We'd only want to use those methods here, - // though. Doesn't seem worth the hassle in the near term. - if err = iter.Close(); err != nil { - rangeDelIter.Close() - rangeDelIter = nil - } - } - if rangeDelIter != nil { - // Ensure that rangeDelIter is not closed until the compaction is - // finished. This is necessary because range tombstone processing - // requires the range tombstones to be held in memory for up to the - // lifetime of the compaction. - c.closers = append(c.closers, rangeDelIter) - rangeDelIter = noCloseIter{rangeDelIter} - - // Truncate the range tombstones returned by the iterator to the - // upper bound of the atomic compaction unit of the file. We want to - // truncate the range tombstone to the bounds of the file, but files - // with split user keys pose an obstacle: The file's largest bound - // is inclusive whereas the range tombstone's end is exclusive. - // - // Consider the example: - // - // 000001:[b-f#200] range del [c,k) - // 000002:[f#190-g#inf] range del [c,k) - // 000003:[g#500-i#3] - // - // Files 000001 and 000002 contain the untruncated range tombstones - // [c,k). While the keyspace covered by 000003 was at one point - // deleted by the tombstone [c,k), the tombstone may have already - // been compacted away and the file does not contain an untruncated - // range tombstone. We want to bound 000001's tombstone to the file - // bounds, but it's not possible to encode a range tombstone with an - // end boundary within a user key (eg, between sequence numbers - // f#200 and f#190). Instead, we expand 000001 to its atomic - // compaction unit (000001 and 000002) and truncate the tombstone to - // g#inf. - // - // NB: We must not use the atomic compaction unit of the entire - // compaction, because the [c,k) tombstone contained in the file - // 000001 ≥ g. If 000001, 000002 and 000003 are all included in the - // same compaction, the compaction's atomic compaction unit includes - // 000003. However 000003's keys must not be covered by 000001's - // untruncated range tombstone. - // - // Note that we need do this truncation at read time in order to - // handle sstables generated by RocksDB and earlier versions of - // Pebble which do not truncate range tombstones to atomic - // compaction unit boundaries at write time. - // - // The current Pebble compaction logic DOES truncate tombstones to - // atomic unit boundaries at compaction time too. - atomicUnit, _ := expandToAtomicUnit(c.cmp, f.Slice(), true /* disableIsCompacting */) - lowerBound, upperBound := manifest.KeyRange(c.cmp, atomicUnit.Iter()) - // Range deletion tombstones are often written to sstables - // untruncated on the end key side. However, they are still only - // valid within a given file's bounds. The logic for writing range - // tombstones to an output file sometimes has an incomplete view - // of range tombstones outside the file's internal key bounds. Skip - // any range tombstones completely outside file bounds. - rangeDelIter = keyspan.Truncate( - c.cmp, rangeDelIter, lowerBound.UserKey, upperBound.UserKey, &f.Smallest, &f.Largest) - } - if rangeDelIter == nil { - rangeDelIter = emptyKeyspanIter - } - return rangeDelIter, err - } - iterOpts := IterOptions{logger: c.logger} - // TODO(bananabrick): Get rid of the extra manifest.Level parameter and fold it into - // compactionLevel. - addItersForLevel := func(level *compactionLevel, l manifest.Level) error { - iters = append(iters, newLevelIter(iterOpts, c.cmp, nil /* split */, newIters, - level.files.Iter(), l, &c.bytesIterated)) - // TODO(jackson): Use keyspan.LevelIter to avoid loading all the range - // deletions into memory upfront. (See #2015, which reverted this.) - // There will be no user keys that are split between sstables - // within a level in Cockroach 23.1, which unblocks this optimization. - // Add the range deletion iterator for each file as an independent level - // in mergingIter, as opposed to making a levelIter out of those. This - // is safer as levelIter expects all keys coming from underlying - // iterators to be in order. Due to compaction / tombstone writing - // logic in finishOutput(), it is possible for range tombstones to not - // be strictly ordered across all files in one level. - // - // Consider this example from the metamorphic tests (also repeated in - // finishOutput()), consisting of three L3 files with their bounds - // specified in square brackets next to the file name: - // - // ./000240.sst [tmgc#391,MERGE-tmgc#391,MERGE] - // tmgc#391,MERGE [786e627a] - // tmgc-udkatvs#331,RANGEDEL - // - // ./000241.sst [tmgc#384,MERGE-tmgc#384,MERGE] - // tmgc#384,MERGE [666c7070] - // tmgc-tvsalezade#383,RANGEDEL - // tmgc-tvsalezade#331,RANGEDEL - // - // ./000242.sst [tmgc#383,RANGEDEL-tvsalezade#72057594037927935,RANGEDEL] - // tmgc-tvsalezade#383,RANGEDEL - // tmgc#375,SET [72646c78766965616c72776865676e79] - // tmgc-tvsalezade#356,RANGEDEL - // - // Here, the range tombstone in 000240.sst falls "after" one in - // 000241.sst, despite 000240.sst being ordered "before" 000241.sst for - // levelIter's purposes. While each file is still consistent before its - // bounds, it's safer to have all rangedel iterators be visible to - // mergingIter. - iter := level.files.Iter() - for f := iter.First(); f != nil; f = iter.Next() { - rangeDelIter, err := newRangeDelIter(iter.Take(), nil, &c.bytesIterated) - if err != nil { - return errors.Wrapf(err, "pebble: could not open table %s", errors.Safe(f.FileNum)) - } - if rangeDelIter != emptyKeyspanIter { + // Populate iters, rangeDelIters and rangeKeyIters with the appropriate + // constituent iterators. This depends on whether this is a flush or a + // compaction. + if len(c.flushing) != 0 { + // If flushing, we need to build the input iterators over the memtables + // stored in c.flushing. + for i := range c.flushing { + f := c.flushing[i] + iters = append(iters, f.newFlushIter(nil, &c.bytesIterated)) + rangeDelIter := f.newRangeDelIter(nil) + if rangeDelIter != nil { rangeDelIters = append(rangeDelIters, rangeDelIter) } - } - - // Check if this level has any range keys. - hasRangeKeys := false - for f := iter.First(); f != nil; f = iter.Next() { - if f.HasRangeKeys { - hasRangeKeys = true - break + if rangeKeyIter := f.newRangeKeyIter(nil); rangeKeyIter != nil { + rangeKeyIters = append(rangeKeyIters, rangeKeyIter) } } - if hasRangeKeys { - li := &keyspan.LevelIter{} - newRangeKeyIterWrapper := func(file *manifest.FileMetadata, iterOptions *keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) { - iter, err := newRangeKeyIter(file, iterOptions) - if iter != nil { + } else { + addItersForLevel := func(level *compactionLevel, l manifest.Level) error { + // Add a *levelIter for point iterators. Because we don't call + // initRangeDel, the levelIter will close and forget the range + // deletion iterator when it steps on to a new file. Surfacing range + // deletions to compactions are handled below. + iters = append(iters, newLevelIter(iterOpts, c.comparer, newIters, + level.files.Iter(), l, internalIterOpts{ + bytesIterated: &c.bytesIterated, + bufferPool: &c.bufferPool, + })) + // TODO(jackson): Use keyspan.LevelIter to avoid loading all the range + // deletions into memory upfront. (See #2015, which reverted this.) + // There will be no user keys that are split between sstables + // within a level in Cockroach 23.1, which unblocks this optimization. + + // Add the range deletion iterator for each file as an independent level + // in mergingIter, as opposed to making a levelIter out of those. This + // is safer as levelIter expects all keys coming from underlying + // iterators to be in order. Due to compaction / tombstone writing + // logic in finishOutput(), it is possible for range tombstones to not + // be strictly ordered across all files in one level. + // + // Consider this example from the metamorphic tests (also repeated in + // finishOutput()), consisting of three L3 files with their bounds + // specified in square brackets next to the file name: + // + // ./000240.sst [tmgc#391,MERGE-tmgc#391,MERGE] + // tmgc#391,MERGE [786e627a] + // tmgc-udkatvs#331,RANGEDEL + // + // ./000241.sst [tmgc#384,MERGE-tmgc#384,MERGE] + // tmgc#384,MERGE [666c7070] + // tmgc-tvsalezade#383,RANGEDEL + // tmgc-tvsalezade#331,RANGEDEL + // + // ./000242.sst [tmgc#383,RANGEDEL-tvsalezade#72057594037927935,RANGEDEL] + // tmgc-tvsalezade#383,RANGEDEL + // tmgc#375,SET [72646c78766965616c72776865676e79] + // tmgc-tvsalezade#356,RANGEDEL + // + // Here, the range tombstone in 000240.sst falls "after" one in + // 000241.sst, despite 000240.sst being ordered "before" 000241.sst for + // levelIter's purposes. While each file is still consistent before its + // bounds, it's safer to have all rangedel iterators be visible to + // mergingIter. + iter := level.files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { + rangeDelIter, closer, err := c.newRangeDelIter(newIters, iter.Take(), nil, l, &c.bytesIterated) + if err != nil { + // The error will already be annotated with the BackingFileNum, so + // we annotate it with the FileNum. + return errors.Wrapf(err, "pebble: could not open table %s", errors.Safe(f.FileNum)) + } + if rangeDelIter == nil { + continue + } + rangeDelIters = append(rangeDelIters, rangeDelIter) + c.closers = append(c.closers, closer) + } + + // Check if this level has any range keys. + hasRangeKeys := false + for f := iter.First(); f != nil; f = iter.Next() { + if f.HasRangeKeys { + hasRangeKeys = true + break + } + } + if hasRangeKeys { + li := &keyspan.LevelIter{} + newRangeKeyIterWrapper := func(file *manifest.FileMetadata, iterOptions keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) { + iter, err := newRangeKeyIter(file, iterOptions) + if err != nil { + return nil, err + } else if iter == nil { + return emptyKeyspanIter, nil + } // Ensure that the range key iter is not closed until the compaction is // finished. This is necessary because range key processing // requires the range keys to be held in memory for up to the @@ -1475,39 +1466,46 @@ func (c *compaction) newInputIter( // range tombstones. This is because range keys were added after we // stopped splitting user keys across sstables, so all the range keys // in this sstable must wholly lie within the file's bounds. + return iter, err } - if iter == nil { - iter = emptyKeyspanIter - } - return iter, err + li.Init(keyspan.SpanIterOptions{}, c.cmp, newRangeKeyIterWrapper, level.files.Iter(), l, manifest.KeyTypeRange) + rangeKeyIters = append(rangeKeyIters, li) } - li.Init(keyspan.SpanIterOptions{}, c.cmp, newRangeKeyIterWrapper, level.files.Iter(), l, manifest.KeyTypeRange) - rangeKeyIters = append(rangeKeyIters, li) + return nil } - return nil - } - if c.startLevel.level != 0 { - if err = addItersForLevel(c.startLevel, manifest.Level(c.startLevel.level)); err != nil { - return nil, err - } - } else { - for _, info := range c.l0SublevelInfo { - if err = addItersForLevel( - &compactionLevel{0, info.LevelSlice}, info.sublevel); err != nil { + for i := range c.inputs { + // If the level is annotated with l0SublevelInfo, expand it into one + // level per sublevel. + // TODO(jackson): Perform this expansion even earlier when we pick the + // compaction? + if len(c.inputs[i].l0SublevelInfo) > 0 { + for _, info := range c.startLevel.l0SublevelInfo { + sublevelCompactionLevel := &compactionLevel{0, info.LevelSlice, nil} + if err := addItersForLevel(sublevelCompactionLevel, info.sublevel); err != nil { + return nil, err + } + } + continue + } + if err := addItersForLevel(&c.inputs[i], manifest.Level(c.inputs[i].level)); err != nil { return nil, err } } } - if len(c.extraLevels) > 0 { - if err = addItersForLevel(c.extraLevels[0], manifest.Level(c.extraLevels[0].level)); err != nil { - return nil, err - } - } - if err = addItersForLevel(c.outputLevel, manifest.Level(c.outputLevel.level)); err != nil { - return nil, err - } + // In normal operation, levelIter iterates over the point operations in a + // level, and initializes a rangeDelIter pointer for the range deletions in + // each table. During compaction, we want to iterate over the merged view of + // point operations and range deletions. In order to do this we create one + // levelIter per level to iterate over the point operations, and collect up + // all the range deletion files. + // + // The range deletion levels are first combined with a keyspan.MergingIter + // (currently wrapped by a keyspan.InternalIteratorShim to satisfy the + // internal iterator interface). The resulting merged rangedel iterator is + // then included with the point levels in a single mergingIter. + // // Combine all the rangedel iterators using a keyspan.MergingIterator and a // InternalIteratorShim so that the range deletions may be interleaved in // the compaction input. @@ -1517,17 +1515,116 @@ func (c *compaction) newInputIter( c.rangeDelIter.Init(c.cmp, rangeDelIters...) iters = append(iters, &c.rangeDelIter) } - pointKeyIter := newMergingIter(c.logger, &c.stats, c.cmp, nil, iters...) + + // If there's only one constituent point iterator, we can avoid the overhead + // of a *mergingIter. This is possible, for example, when performing a flush + // of a single memtable. Otherwise, combine all the iterators into a merging + // iter. + iter := iters[0] + if len(iters) > 0 { + iter = newMergingIter(c.logger, &c.stats, c.cmp, nil, iters...) + } + // If there are range key iterators, we need to combine them using + // keyspan.MergingIter, and then interleave them among the points. if len(rangeKeyIters) > 0 { mi := &keyspan.MergingIter{} mi.Init(c.cmp, rangeKeyCompactionTransform(c.equal, snapshots, c.elideRangeKey), new(keyspan.MergingBuffers), rangeKeyIters...) di := &keyspan.DefragmentingIter{} di.Init(c.comparer, mi, keyspan.DefragmentInternal, keyspan.StaticDefragmentReducer, new(keyspan.DefragmentingBuffers)) - c.rangeKeyInterleaving.Init(c.comparer, pointKeyIter, di, nil /* hooks */, nil /* lowerBound */, nil /* upperBound */) - return &c.rangeKeyInterleaving, nil + c.rangeKeyInterleaving.Init(c.comparer, iter, di, keyspan.InterleavingIterOpts{}) + iter = &c.rangeKeyInterleaving + } + return iter, nil +} + +func (c *compaction) newRangeDelIter( + newIters tableNewIters, + f manifest.LevelFile, + _ *IterOptions, + l manifest.Level, + bytesIterated *uint64, +) (keyspan.FragmentIterator, io.Closer, error) { + iter, rangeDelIter, err := newIters(context.Background(), f.FileMetadata, + &IterOptions{level: l}, internalIterOpts{ + bytesIterated: &c.bytesIterated, + bufferPool: &c.bufferPool, + }) + if err != nil { + return nil, nil, err + } + // TODO(peter): It is mildly wasteful to open the point iterator only to + // immediately close it. One way to solve this would be to add new + // methods to tableCache for creating point and range-deletion iterators + // independently. We'd only want to use those methods here, + // though. Doesn't seem worth the hassle in the near term. + if err = iter.Close(); err != nil { + if rangeDelIter != nil { + err = errors.CombineErrors(err, rangeDelIter.Close()) + } + return nil, nil, err + } + if rangeDelIter == nil { + // The file doesn't contain any range deletions. + return nil, nil, nil } - return pointKeyIter, nil + // Ensure that rangeDelIter is not closed until the compaction is + // finished. This is necessary because range tombstone processing + // requires the range tombstones to be held in memory for up to the + // lifetime of the compaction. + closer := rangeDelIter + rangeDelIter = noCloseIter{rangeDelIter} + + // Truncate the range tombstones returned by the iterator to the + // upper bound of the atomic compaction unit of the file. We want to + // truncate the range tombstone to the bounds of the file, but files + // with split user keys pose an obstacle: The file's largest bound + // is inclusive whereas the range tombstone's end is exclusive. + // + // Consider the example: + // + // 000001:[b-f#200] range del [c,k) + // 000002:[f#190-g#inf] range del [c,k) + // 000003:[g#500-i#3] + // + // Files 000001 and 000002 contain the untruncated range tombstones + // [c,k). While the keyspace covered by 000003 was at one point + // deleted by the tombstone [c,k), the tombstone may have already + // been compacted away and the file does not contain an untruncated + // range tombstone. We want to bound 000001's tombstone to the file + // bounds, but it's not possible to encode a range tombstone with an + // end boundary within a user key (eg, between sequence numbers + // f#200 and f#190). Instead, we expand 000001 to its atomic + // compaction unit (000001 and 000002) and truncate the tombstone to + // g#inf. + // + // NB: We must not use the atomic compaction unit of the entire + // compaction, because the [c,k) tombstone contained in the file + // 000001 ≥ g. If 000001, 000002 and 000003 are all included in the + // same compaction, the compaction's atomic compaction unit includes + // 000003. However 000003's keys must not be covered by 000001's + // untruncated range tombstone. + // + // Note that we need do this truncation at read time in order to + // handle sstables generated by RocksDB and earlier versions of + // Pebble which do not truncate range tombstones to atomic + // compaction unit boundaries at write time. + // + // The current Pebble compaction logic DOES truncate tombstones to + // atomic unit boundaries at compaction time too. + atomicUnit, _ := expandToAtomicUnit(c.cmp, f.Slice(), true /* disableIsCompacting */) + lowerBound, upperBound := manifest.KeyRange(c.cmp, atomicUnit.Iter()) + // Range deletion tombstones are often written to sstables + // untruncated on the end key side. However, they are still only + // valid within a given file's bounds. The logic for writing range + // tombstones to an output file sometimes has an incomplete view + // of range tombstones outside the file's internal key bounds. Skip + // any range tombstones completely outside file bounds. + rangeDelIter = keyspan.Truncate( + c.cmp, rangeDelIter, lowerBound.UserKey, upperBound.UserKey, + &f.Smallest, &f.Largest, false, /* panicOnUpperTruncate */ + ) + return rangeDelIter, closer, nil } func (c *compaction) String() string { @@ -1602,31 +1699,6 @@ func (d *DB) addInProgressCompaction(c *compaction) { d.opts.Logger.Fatalf("could not update state for compaction: %s", err) } } - - if false { - // TODO(peter): Do we want to keep this? It is useful for seeing the - // concurrent compactions/flushes that are taking place. Right now, this - // spams the logs and output to tests. Figure out a way to useful expose - // it. - strs := make([]string, 0, len(d.mu.compact.inProgress)) - for c := range d.mu.compact.inProgress { - var s string - if c.startLevel.level == -1 { - s = fmt.Sprintf("mem->L%d", c.outputLevel.level) - } else { - s = fmt.Sprintf("L%d->L%d:%.1f", c.startLevel.level, c.outputLevel.level, c.score) - } - strs = append(strs, s) - } - // This odd sorting function is intended to sort "mem" before "L*". - sort.Slice(strs, func(i, j int) bool { - if strs[i][0] == strs[j][0] { - return strs[i] < strs[j] - } - return strs[i] > strs[j] - }) - d.opts.Logger.Infof("compactions: %s", strings.Join(strs, " ")) - } } // Removes compaction markers from files in a compaction. The rollback parameter @@ -1636,7 +1708,8 @@ func (d *DB) addInProgressCompaction(c *compaction) { // DB.mu must be held when calling this method, however this method can drop and // re-acquire that mutex. All writes to the manifest for this compaction should // have completed by this point. -func (d *DB) removeInProgressCompaction(c *compaction, rollback bool) { +func (d *DB) clearCompactingState(c *compaction, rollback bool) { + c.versionEditApplied = true for _, cl := range c.inputs { iter := cl.files.Iter() for f := iter.First(); f != nil; f = iter.Next() { @@ -1660,8 +1733,6 @@ func (d *DB) removeInProgressCompaction(c *compaction, rollback bool) { f.IsIntraL0Compacting = false } } - delete(d.mu.compact.inProgress, c) - l0InProgress := inProgressL0Compactions(d.getInProgressCompactionInfoLocked(c)) func() { // InitCompactingFileInfo requires that no other manifest writes be @@ -1678,16 +1749,12 @@ func (d *DB) removeInProgressCompaction(c *compaction, rollback bool) { func (d *DB) calculateDiskAvailableBytes() uint64 { if space, err := d.opts.FS.GetDiskUsage(d.dirname); err == nil { - atomic.StoreUint64(&d.atomic.diskAvailBytes, space.AvailBytes) + d.diskAvailBytes.Store(space.AvailBytes) return space.AvailBytes } else if !errors.Is(err, vfs.ErrUnsupported) { d.opts.EventListener.BackgroundError(err) } - return atomic.LoadUint64(&d.atomic.diskAvailBytes) -} - -func (d *DB) getDiskAvailableBytesCached() uint64 { - return atomic.LoadUint64(&d.atomic.diskAvailBytes) + return d.diskAvailBytes.Load() } func (d *DB) getDeletionPacerInfo() deletionPacerInfo { @@ -1704,6 +1771,14 @@ func (d *DB) getDeletionPacerInfo() deletionPacerInfo { return pacerInfo } +// onObsoleteTableDelete is called to update metrics when an sstable is deleted. +func (d *DB) onObsoleteTableDelete(fileSize uint64) { + d.mu.Lock() + d.mu.versions.metrics.Table.ObsoleteCount-- + d.mu.versions.metrics.Table.ObsoleteSize -= fileSize + d.mu.Unlock() +} + // maybeScheduleFlush schedules a flush if necessary. // // d.mu must be held when calling this. @@ -1733,7 +1808,7 @@ func (d *DB) passedFlushThreshold() bool { if d.mu.mem.queue[n].flushForced { // A flush was forced. Pretend the memtable size is the configured // size. See minFlushSize below. - size += uint64(d.opts.MemTableSize) + size += d.opts.MemTableSize } else { size += d.mu.mem.queue[n].totalBytes() } @@ -1747,7 +1822,7 @@ func (d *DB) passedFlushThreshold() bool { // configured memtable size. This prevents flushing of memtables at startup // while we're undergoing the ramp period on the memtable size. See // DB.newMemTable(). - minFlushSize := uint64(d.opts.MemTableSize) / 2 + minFlushSize := d.opts.MemTableSize / 2 return size >= minFlushSize } @@ -1797,8 +1872,8 @@ func (d *DB) maybeScheduleDelayedFlush(tbl *memTable, dur time.Duration) { d.makeRoomForWrite(nil) } else { mem.flushForced = true - d.maybeScheduleFlush() } + d.maybeScheduleFlush() } }() } @@ -1850,26 +1925,58 @@ func (d *DB) runIngestFlush(c *compaction) (*manifest.VersionEdit, error) { ve := &versionEdit{} var level int var err error + var fileToSplit *fileMetadata + var ingestSplitFiles []ingestSplitFile for _, file := range c.flushing[0].flushable.(*ingestedFlushable).files { - level, err = ingestTargetLevel( - d.newIters, d.tableNewRangeKeyIter, iterOpts, d.cmp, - c.version, baseLevel, d.mu.compact.inProgress, file, + suggestSplit := d.opts.Experimental.IngestSplit != nil && d.opts.Experimental.IngestSplit() && + d.FormatMajorVersion() >= FormatVirtualSSTables + level, fileToSplit, err = ingestTargetLevel( + d.newIters, d.tableNewRangeKeyIter, iterOpts, d.opts.Comparer, + c.version, baseLevel, d.mu.compact.inProgress, file.FileMetadata, + suggestSplit, ) if err != nil { return nil, err } - ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: level, Meta: file}) + ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: level, Meta: file.FileMetadata}) + if fileToSplit != nil { + ingestSplitFiles = append(ingestSplitFiles, ingestSplitFile{ + ingestFile: file.FileMetadata, + splitFile: fileToSplit, + level: level, + }) + } levelMetrics := c.metrics[level] if levelMetrics == nil { levelMetrics = &LevelMetrics{} c.metrics[level] = levelMetrics } - levelMetrics.NumFiles++ - levelMetrics.Size += int64(file.Size) levelMetrics.BytesIngested += file.Size levelMetrics.TablesIngested++ } + updateLevelMetricsOnExcise := func(m *fileMetadata, level int, added []newFileEntry) { + levelMetrics := c.metrics[level] + if levelMetrics == nil { + levelMetrics = &LevelMetrics{} + c.metrics[level] = levelMetrics + } + levelMetrics.NumFiles-- + levelMetrics.Size -= int64(m.Size) + for i := range added { + levelMetrics.NumFiles++ + levelMetrics.Size += int64(added[i].Meta.Size) + } + } + + if len(ingestSplitFiles) > 0 { + ve.DeletedFiles = make(map[manifest.DeletedFileEntry]*manifest.FileMetadata) + replacedFiles := make(map[base.FileNum][]newFileEntry) + if err := d.ingestSplit(ve, updateLevelMetricsOnExcise, ingestSplitFiles, replacedFiles); err != nil { + return nil, err + } + } + return ve, nil } @@ -1909,6 +2016,7 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { // flushable batch in the same flush, since the memtable and flushableBatch // have the same logNum, the logNum invariant check below will trigger. var n, inputs int + var inputBytes uint64 var ingest bool for ; n < len(d.mu.mem.queue)-1; n++ { if f, ok := d.mu.mem.queue[n].flushable.(*ingestedFlushable); ok { @@ -1937,6 +2045,7 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { if !d.mu.mem.queue[n].readyForFlush() { break } + inputBytes += d.mu.mem.queue[n].inuseBytes() } if n == 0 { // None of the immutable memtables are ready for flushing. @@ -1963,20 +2072,21 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { } c := newFlush(d.opts, d.mu.versions.currentVersion(), - d.mu.versions.picker.getBaseLevel(), d.mu.mem.queue[:n]) + d.mu.versions.picker.getBaseLevel(), d.mu.mem.queue[:n], d.timeNow()) d.addInProgressCompaction(c) jobID := d.mu.nextJobID d.mu.nextJobID++ d.opts.EventListener.FlushBegin(FlushInfo{ - JobID: jobID, - Input: inputs, - Ingest: ingest, + JobID: jobID, + Input: inputs, + InputBytes: inputBytes, + Ingest: ingest, }) startTime := d.timeNow() var ve *manifest.VersionEdit - var pendingOutputs []*manifest.FileMetadata + var pendingOutputs []physicalMeta var stats compactStats // To determine the target level of the files in the ingestedFlushable, we // need to acquire the logLock, and not release it for that duration. Since, @@ -1997,12 +2107,13 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { } info := FlushInfo{ - JobID: jobID, - Input: inputs, - Duration: d.timeNow().Sub(startTime), - Done: true, - Ingest: ingest, - Err: err, + JobID: jobID, + Input: inputs, + InputBytes: inputBytes, + Duration: d.timeNow().Sub(startTime), + Done: true, + Ingest: ingest, + Err: err, } if err == nil { for i := range ve.NewFiles { @@ -2036,13 +2147,42 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { metrics.BytesIn += d.mu.mem.queue[i].logSize } } + } else if len(ve.DeletedFiles) > 0 { + // c.kind == compactionKindIngestedFlushable && we have deleted files due + // to ingest-time splits. + // + // Iterate through all other compactions, and check if their inputs have + // been replaced due to an ingest-time split. In that case, cancel the + // compaction. + for c2 := range d.mu.compact.inProgress { + for i := range c2.inputs { + iter := c2.inputs[i].files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { + if _, ok := ve.DeletedFiles[deletedFileEntry{FileNum: f.FileNum, Level: c2.inputs[i].level}]; ok { + c2.cancel.Store(true) + break + } + } + } + } } err = d.mu.versions.logAndApply(jobID, ve, c.metrics, false, /* forceRotation */ func() []compactionInfo { return d.getInProgressCompactionInfoLocked(c) }) if err != nil { info.Err = err // TODO(peter): untested. - d.mu.versions.obsoleteTables = append(d.mu.versions.obsoleteTables, pendingOutputs...) + for _, f := range pendingOutputs { + // Note that the FileBacking for the file metadata might not have + // been set yet. So, we directly use the FileNum. Since these + // files were generated as compaction outputs, these must be + // physical files on disk. This property might not hold once + // https://github.com/cockroachdb/pebble/issues/389 is + // implemented if #389 creates virtual sstables as output files. + d.mu.versions.obsoleteTables = append( + d.mu.versions.obsoleteTables, + fileInfo{f.FileNum.DiskFileNum(), f.Size}, + ) + } d.mu.versions.updateObsoleteTableMetricsLocked() } } else { @@ -2052,12 +2192,19 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { } bytesFlushed = c.bytesIterated - d.mu.snapshots.cumulativePinnedCount += stats.cumulativePinnedKeys - d.mu.snapshots.cumulativePinnedSize += stats.cumulativePinnedSize - d.maybeUpdateDeleteCompactionHints(c) - d.removeInProgressCompaction(c, err != nil) - d.mu.versions.incrementCompactions(c.kind, c.extraLevels) + // If err != nil, then the flush will be retried, and we will recalculate + // these metrics. + if err == nil { + d.mu.snapshots.cumulativePinnedCount += stats.cumulativePinnedKeys + d.mu.snapshots.cumulativePinnedSize += stats.cumulativePinnedSize + d.mu.versions.metrics.Keys.MissizedTombstonesCount += stats.countMissizedDels + d.maybeUpdateDeleteCompactionHints(c) + } + + d.clearCompactingState(c, err != nil) + delete(d.mu.compact.inProgress, c) + d.mu.versions.incrementCompactions(c.kind, c.extraLevels, c.pickerMetrics) var flushed flushableList if err == nil { @@ -2072,6 +2219,36 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { d.mu.versions.metrics.Flush.AsIngestTableCount += l.TablesIngested } } + + // Update if any eventually file-only snapshots have now transitioned to + // being file-only. + earliestUnflushedSeqNum := d.getEarliestUnflushedSeqNumLocked() + currentVersion := d.mu.versions.currentVersion() + for s := d.mu.snapshots.root.next; s != &d.mu.snapshots.root; { + if s.efos == nil { + s = s.next + continue + } + if base.Visible(earliestUnflushedSeqNum, s.efos.seqNum, InternalKeySeqNumMax) { + s = s.next + continue + } + if s.efos.excised.Load() { + // If a concurrent excise has happened that overlaps with one of the key + // ranges this snapshot is interested in, this EFOS cannot transition to + // a file-only snapshot as keys in that range could now be deleted. Move + // onto the next snapshot. + s = s.next + continue + } + currentVersion.Ref() + + // NB: s.efos.transitionToFileOnlySnapshot could close s, in which + // case s.next would be nil. Save it before calling it. + next := s.next + _ = s.efos.transitionToFileOnlySnapshot(currentVersion) + s = next + } } // Signal FlushEnd after installing the new readState. This helps for unit // tests that use the callback to trigger a read using an iterator with @@ -2088,14 +2265,9 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { flushed[i].readerUnrefLocked(true) } - d.deleteObsoleteFiles(jobID, false /* waitForOngoing */) + d.deleteObsoleteFiles(jobID) - // Mark all the memtables we flushed as flushed. Note that we do this last so - // that a synchronous call to DB.Flush() will not return until the deletion - // of obsolete files from this job have completed. This makes testing easier - // and provides similar behavior to manual compactions where the compaction - // is not marked as completed until the deletion of obsolete files job has - // completed. + // Mark all the memtables we flushed as flushed. for i := range flushed { close(flushed[i].flushed) } @@ -2164,6 +2336,7 @@ func (d *DB) maybeScheduleCompactionPicker( } env := compactionEnv{ + diskAvailBytes: d.diskAvailBytes.Load(), earliestSnapshotSeqNum: d.mu.snapshots.earliest(), earliestUnflushedSeqNum: d.getEarliestUnflushedSeqNumLocked(), } @@ -2180,7 +2353,7 @@ func (d *DB) maybeScheduleCompactionPicker( d.mu.compact.deletionHints = unresolvedHints if len(inputs) > 0 { - c := newDeleteOnlyCompaction(d.opts, v, inputs) + c := newDeleteOnlyCompaction(d.opts, v, inputs, d.timeNow()) d.mu.compact.compactingCount++ d.addInProgressCompaction(c) go d.compact(c, nil) @@ -2188,11 +2361,12 @@ func (d *DB) maybeScheduleCompactionPicker( } for len(d.mu.compact.manual) > 0 && d.mu.compact.compactingCount < maxConcurrentCompactions { + v := d.mu.versions.currentVersion() manual := d.mu.compact.manual[0] env.inProgressCompactions = d.getInProgressCompactionInfoLocked(nil) - pc, retryLater := d.mu.versions.picker.pickManual(env, manual) + pc, retryLater := pickManualCompaction(v, d.opts, env, d.mu.versions.picker.getBaseLevel(), manual) if pc != nil { - c := newCompaction(pc, d.opts) + c := newCompaction(pc, d.opts, d.timeNow(), d.ObjProvider()) d.mu.compact.manual = d.mu.compact.manual[1:] d.mu.compact.compactingCount++ d.addInProgressCompaction(c) @@ -2219,7 +2393,7 @@ func (d *DB) maybeScheduleCompactionPicker( if pc == nil { break } - c := newCompaction(pc, d.opts) + c := newCompaction(pc, d.opts, d.timeNow(), d.ObjProvider()) d.mu.compact.compactingCount++ d.addInProgressCompaction(c) go d.compact(c, nil) @@ -2509,6 +2683,13 @@ func (d *DB) compact(c *compaction, errChannel chan error) { d.opts.EventListener.BackgroundError(err) } d.mu.compact.compactingCount-- + delete(d.mu.compact.inProgress, c) + // Add this compaction's duration to the cumulative duration. NB: This + // must be atomic with the above removal of c from + // d.mu.compact.InProgress to ensure Metrics.Compact.Duration does not + // miss or double count a completing compaction's duration. + d.mu.compact.duration += d.timeNow().Sub(c.beganAt) + // The previous compaction may have produced too many files in a // level, so reschedule another compaction if needed. d.maybeScheduleCompaction() @@ -2537,13 +2718,40 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) { info.Duration = d.timeNow().Sub(startTime) if err == nil { - d.mu.versions.logLock() - err = d.mu.versions.logAndApply(jobID, ve, c.metrics, false /* forceRotation */, func() []compactionInfo { - return d.getInProgressCompactionInfoLocked(c) - }) + err = func() error { + var err error + d.mu.versions.logLock() + // Check if this compaction had a conflicting operation (eg. a d.excise()) + // that necessitates it restarting from scratch. Note that since we hold + // the manifest lock, we don't expect this bool to change its value + // as only the holder of the manifest lock will ever write to it. + if c.cancel.Load() { + err = firstError(err, ErrCancelledCompaction) + } + if err != nil { + // logAndApply calls logUnlock. If we didn't call it, we need to call + // logUnlock ourselves. + d.mu.versions.logUnlock() + return err + } + return d.mu.versions.logAndApply(jobID, ve, c.metrics, false /* forceRotation */, func() []compactionInfo { + return d.getInProgressCompactionInfoLocked(c) + }) + }() if err != nil { // TODO(peter): untested. - d.mu.versions.obsoleteTables = append(d.mu.versions.obsoleteTables, pendingOutputs...) + for _, f := range pendingOutputs { + // Note that the FileBacking for the file metadata might not have + // been set yet. So, we directly use the FileNum. Since these + // files were generated as compaction outputs, these must be + // physical files on disk. This property might not hold once + // https://github.com/cockroachdb/pebble/issues/389 is + // implemented if #389 creates virtual sstables as output files. + d.mu.versions.obsoleteTables = append( + d.mu.versions.obsoleteTables, + fileInfo{f.FileNum.DiskFileNum(), f.Size}, + ) + } d.mu.versions.updateObsoleteTableMetricsLocked() } } @@ -2555,17 +2763,19 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) { e := &ve.NewFiles[i] info.Output.Tables = append(info.Output.Tables, e.Meta.TableInfo()) } + d.mu.snapshots.cumulativePinnedCount += stats.cumulativePinnedKeys + d.mu.snapshots.cumulativePinnedSize += stats.cumulativePinnedSize + d.mu.versions.metrics.Keys.MissizedTombstonesCount += stats.countMissizedDels + d.maybeUpdateDeleteCompactionHints(c) } - d.mu.snapshots.cumulativePinnedCount += stats.cumulativePinnedKeys - d.mu.snapshots.cumulativePinnedSize += stats.cumulativePinnedSize - - d.maybeUpdateDeleteCompactionHints(c) - d.removeInProgressCompaction(c, err != nil) - d.mu.versions.incrementCompactions(c.kind, c.extraLevels) + // NB: clearing compacting state must occur before updating the read state; + // L0Sublevels initialization depends on it. + d.clearCompactingState(c, err != nil) + d.mu.versions.incrementCompactions(c.kind, c.extraLevels, c.pickerMetrics) d.mu.versions.incrementCompactionBytes(-c.bytesWritten) - info.TotalDuration = d.timeNow().Sub(startTime) + info.TotalDuration = d.timeNow().Sub(c.beganAt) d.opts.EventListener.CompactionEnd(info) // Update the read state before deleting obsolete files because the @@ -2576,7 +2786,7 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) { d.updateReadStateLocked(d.opts.DebugCheck) d.updateTableStatsLocked(ve.NewFiles) } - d.deleteObsoleteFiles(jobID, true /* waitForOngoing */) + d.deleteObsoleteFiles(jobID) return err } @@ -2584,6 +2794,78 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) { type compactStats struct { cumulativePinnedKeys uint64 cumulativePinnedSize uint64 + countMissizedDels uint64 +} + +// runCopyCompaction runs a copy compaction where a new FileNum is created that +// is a byte-for-byte copy of the input file. This is used in lieu of a move +// compaction when a file is being moved across the local/remote storage +// boundary. +// +// d.mu must be held when calling this method. +func (d *DB) runCopyCompaction( + jobID int, + c *compaction, + meta *fileMetadata, + objMeta objstorage.ObjectMetadata, + versionEdit *versionEdit, +) (ve *versionEdit, pendingOutputs []physicalMeta, retErr error) { + ve = versionEdit + if objMeta.IsRemote() || !remote.ShouldCreateShared(d.opts.Experimental.CreateOnShared, c.outputLevel.level) { + panic("pebble: scheduled a copy compaction that is not actually moving files to shared storage") + } + // Note that based on logic in the compaction picker, we're guaranteed + // meta.Virtual is false. + if meta.Virtual { + panic(errors.AssertionFailedf("cannot do a copy compaction of a virtual sstable across local/remote storage")) + } + // We are in the relatively more complex case where we need to copy this + // file to remote/shared storage. Drop the db mutex while we do the + // copy. + // + // To ease up cleanup of the local file and tracking of refs, we create + // a new FileNum. This has the potential of making the block cache less + // effective, however. + metaCopy := new(fileMetadata) + *metaCopy = fileMetadata{ + Size: meta.Size, + CreationTime: meta.CreationTime, + SmallestSeqNum: meta.SmallestSeqNum, + LargestSeqNum: meta.LargestSeqNum, + Stats: meta.Stats, + Virtual: meta.Virtual, + } + if meta.HasPointKeys { + metaCopy.ExtendPointKeyBounds(c.cmp, meta.SmallestPointKey, meta.LargestPointKey) + } + if meta.HasRangeKeys { + metaCopy.ExtendRangeKeyBounds(c.cmp, meta.SmallestRangeKey, meta.LargestRangeKey) + } + metaCopy.FileNum = d.mu.versions.getNextFileNum() + metaCopy.InitPhysicalBacking() + c.metrics = map[int]*LevelMetrics{ + c.outputLevel.level: { + BytesIn: meta.Size, + BytesCompacted: meta.Size, + TablesCompacted: 1, + }, + } + pendingOutputs = append(pendingOutputs, metaCopy.PhysicalMeta()) + + d.mu.Unlock() + defer d.mu.Lock() + _, err := d.objProvider.LinkOrCopyFromLocal(context.TODO(), d.opts.FS, + d.objProvider.Path(objMeta), fileTypeTable, metaCopy.FileBacking.DiskFileNum, + objstorage.CreateOptions{PreferSharedStorage: true}) + if err != nil { + return ve, pendingOutputs, err + } + ve.NewFiles[0].Meta = metaCopy + + if err := d.objProvider.Sync(); err != nil { + return nil, pendingOutputs, err + } + return ve, pendingOutputs, nil } // runCompactions runs a compaction that produces new on-disk tables from @@ -2593,7 +2875,7 @@ type compactStats struct { // re-acquired during the course of this method. func (d *DB) runCompaction( jobID int, c *compaction, -) (ve *versionEdit, pendingOutputs []*fileMetadata, stats compactStats, retErr error) { +) (ve *versionEdit, pendingOutputs []physicalMeta, stats compactStats, retErr error) { // As a sanity check, confirm that the smallest / largest keys for new and // deleted files in the new versionEdit pass a validation function before // returning the edit. @@ -2620,8 +2902,6 @@ func (d *DB) runCompaction( levelMetrics := &LevelMetrics{} iter := cl.files.Iter() for f := iter.First(); f != nil; f = iter.Next() { - levelMetrics.NumFiles-- - levelMetrics.Size -= int64(f.Size) ve.DeletedFiles[deletedFileEntry{ Level: cl.level, FileNum: f.FileNum, @@ -2636,21 +2916,24 @@ func (d *DB) runCompaction( panic("pebble: runCompaction cannot handle compactionKindIngestedFlushable.") } - // Check for a trivial move of one table from one level to the next. We avoid + // Check for a move or copy of one table from one level to the next. We avoid // such a move if there is lots of overlapping grandparent data. Otherwise, // the move could create a parent file that will require a very expensive // merge later on. - if c.kind == compactionKindMove { + if c.kind == compactionKindMove || c.kind == compactionKindCopy { iter := c.startLevel.files.Iter() meta := iter.First() + if invariants.Enabled { + if iter.Next() != nil { + panic("got more than one file for a move or copy compaction") + } + } + objMeta, err := d.objProvider.Lookup(fileTypeTable, meta.FileBacking.DiskFileNum) + if err != nil { + return ve, pendingOutputs, stats, err + } c.metrics = map[int]*LevelMetrics{ - c.startLevel.level: { - NumFiles: -1, - Size: -int64(meta.Size), - }, c.outputLevel.level: { - NumFiles: 1, - Size: int64(meta.Size), BytesMoved: meta.Size, TablesMoved: 1, }, @@ -2663,6 +2946,12 @@ func (d *DB) runCompaction( {Level: c.outputLevel.level, Meta: meta}, }, } + if c.kind == compactionKindCopy { + ve, pendingOutputs, retErr = d.runCopyCompaction(jobID, c, meta, objMeta, ve) + if retErr != nil { + return ve, pendingOutputs, stats, retErr + } + } return ve, nil, stats, nil } @@ -2675,22 +2964,67 @@ func (d *DB) runCompaction( snapshots := d.mu.snapshots.toSlice() formatVers := d.FormatMajorVersion() + if c.flushing == nil { + // Before dropping the db mutex, grab a ref to the current version. This + // prevents any concurrent excises from deleting files that this compaction + // needs to read/maintain a reference to. + // + // Note that unlike user iterators, compactionIter does not maintain a ref + // of the version or read state. + vers := d.mu.versions.currentVersion() + vers.Ref() + defer vers.UnrefLocked() + } + + if c.cancel.Load() { + return ve, nil, stats, ErrCancelledCompaction + } + // Release the d.mu lock while doing I/O. // Note the unusual order: Unlock and then Lock. d.mu.Unlock() defer d.mu.Lock() + // Compactions use a pool of buffers to read blocks, avoiding polluting the + // block cache with blocks that will not be read again. We initialize the + // buffer pool with a size 12. This initial size does not need to be + // accurate, because the pool will grow to accommodate the maximum number of + // blocks allocated at a given time over the course of the compaction. But + // choosing a size larger than that working set avoids any additional + // allocations to grow the size of the pool over the course of iteration. + // + // Justification for initial size 12: In a two-level compaction, at any + // given moment we'll have 2 index blocks in-use and 2 data blocks in-use. + // Additionally, when decoding a compressed block, we'll temporarily + // allocate 1 additional block to hold the compressed buffer. In the worst + // case that all input sstables have two-level index blocks (+2), value + // blocks (+2), range deletion blocks (+n) and range key blocks (+n), we'll + // additionally require 2n+4 blocks where n is the number of input sstables. + // Range deletion and range key blocks are relatively rare, and the cost of + // an additional allocation or two over the course of the compaction is + // considered to be okay. A larger initial size would cause the pool to hold + // on to more memory, even when it's not in-use because the pool will + // recycle buffers up to the current capacity of the pool. The memory use of + // a 12-buffer pool is expected to be within reason, even if all the buffers + // grow to the typical size of an index block (256 KiB) which would + // translate to 3 MiB per compaction. + c.bufferPool.Init(12) + defer c.bufferPool.Release() + iiter, err := c.newInputIter(d.newIters, d.tableNewRangeKeyIter, snapshots) if err != nil { return nil, pendingOutputs, stats, err } c.allowedZeroSeqNum = c.allowZeroSeqNum() + iiter = invalidating.MaybeWrapIfInvariants(iiter) iter := newCompactionIter(c.cmp, c.equal, c.formatKey, d.merge, iiter, snapshots, &c.rangeDelFrag, &c.rangeKeyFrag, c.allowedZeroSeqNum, c.elideTombstone, - c.elideRangeTombstone, d.FormatMajorVersion()) + c.elideRangeTombstone, d.opts.Experimental.IneffectualSingleDeleteCallback, + d.opts.Experimental.SingleDeleteInvariantViolationCallback, + d.FormatMajorVersion()) var ( - createdFiles []base.FileNum + createdFiles []base.DiskFileNum tw *sstable.Writer pinnedKeySize uint64 pinnedValueSize uint64 @@ -2717,8 +3051,9 @@ func (d *DB) runCompaction( DeletedFiles: map[deletedFileEntry]*fileMetadata{}, } + startLevelBytes := c.startLevel.files.SizeSum() outputMetrics := &LevelMetrics{ - BytesIn: c.startLevel.files.SizeSum(), + BytesIn: startLevelBytes, BytesRead: c.outputLevel.files.SizeSum(), } if len(c.extraLevels) > 0 { @@ -2734,22 +3069,24 @@ func (d *DB) runCompaction( } if len(c.extraLevels) > 0 { c.metrics[c.extraLevels[0].level] = &LevelMetrics{} + outputMetrics.MultiLevel.BytesInTop = startLevelBytes + outputMetrics.MultiLevel.BytesIn = outputMetrics.BytesIn + outputMetrics.MultiLevel.BytesRead = outputMetrics.BytesRead } // The table is typically written at the maximum allowable format implied by // the current format major version of the DB. tableFormat := formatVers.MaxTableFormat() - if tableFormat > sstable.TableFormatPebblev3 { - // Since TableFormatPebblev3 does not currently subsume - // TableFormatPebblev2, this panic ensures that we have carefully thought - // through what we are doing before we introduce a format beyond - // TableFormatPebblev3. - panic("cannot handle table format beyond TableFormatPebblev3") - } + + // In format major versions with maximum table formats of Pebblev3, value + // blocks were conditional on an experimental setting. In format major + // versions with maximum table formats of Pebblev4 and higher, value blocks + // are always enabled. if tableFormat == sstable.TableFormatPebblev3 && (d.opts.Experimental.EnableValueBlocks == nil || !d.opts.Experimental.EnableValueBlocks()) { tableFormat = sstable.TableFormatPebblev2 } + writerOpts := d.opts.MakeWriterOptions(c.outputLevel.level, tableFormat) if formatVers < FormatBlockPropertyCollector { // Cannot yet write block properties. @@ -2773,14 +3110,34 @@ func (d *DB) runCompaction( }() newOutput := func() error { + // Check if we've been cancelled by a concurrent operation. + if c.cancel.Load() { + return ErrCancelledCompaction + } fileMeta := &fileMetadata{} d.mu.Lock() fileNum := d.mu.versions.getNextFileNum() fileMeta.FileNum = fileNum - pendingOutputs = append(pendingOutputs, fileMeta) + pendingOutputs = append(pendingOutputs, fileMeta.PhysicalMeta()) d.mu.Unlock() - writable, objMeta, err := d.objProvider.Create(fileTypeTable, fileNum, objstorage.CreateOptions{} /* TODO */) + ctx := context.TODO() + if objiotracing.Enabled { + ctx = objiotracing.WithLevel(ctx, c.outputLevel.level) + switch c.kind { + case compactionKindFlush: + ctx = objiotracing.WithReason(ctx, objiotracing.ForFlush) + case compactionKindIngestedFlushable: + ctx = objiotracing.WithReason(ctx, objiotracing.ForIngestion) + default: + ctx = objiotracing.WithReason(ctx, objiotracing.ForCompaction) + } + } + // Prefer shared storage if present. + createOpts := objstorage.CreateOptions{ + PreferSharedStorage: remote.ShouldCreateShared(d.opts.Experimental.CreateOnShared, c.outputLevel.level), + } + writable, objMeta, err := d.objProvider.Create(ctx, fileTypeTable, fileNum.DiskFileNum(), createOpts) if err != nil { return err } @@ -2802,8 +3159,8 @@ func (d *DB) runCompaction( written: &c.bytesWritten, } } - createdFiles = append(createdFiles, fileNum) - cacheOpts := private.SSTableCacheOpts(d.cacheID, fileNum).(sstable.WriterOption) + createdFiles = append(createdFiles, fileNum.DiskFileNum()) + cacheOpts := private.SSTableCacheOpts(d.cacheID, fileNum.DiskFileNum()).(sstable.WriterOption) const MaxFileWriteAdditionalCPUTime = time.Millisecond * 100 cpuWorkHandle = d.opts.Experimental.CPUWorkPermissionGranter.GetPermission( @@ -2944,9 +3301,13 @@ func (d *DB) runCompaction( meta.Size = writerMeta.Size meta.SmallestSeqNum = writerMeta.SmallestSeqNum meta.LargestSeqNum = writerMeta.LargestSeqNum + meta.InitPhysicalBacking() + // If the file didn't contain any range deletions, we can fill its // table stats now, avoiding unnecessarily loading the table later. - maybeSetStatsFromProperties(meta, &writerMeta.Properties) + maybeSetStatsFromProperties( + meta.PhysicalMeta(), &writerMeta.Properties, + ) if c.flushing == nil { outputMetrics.TablesCompacted++ @@ -3175,7 +3536,7 @@ func (d *DB) runCompaction( return nil, pendingOutputs, stats, err } } - if err := tw.Add(*key, val); err != nil { + if err := tw.AddWithForceObsolete(*key, val, iter.forceObsoleteDueToRangeDel); err != nil { return nil, pendingOutputs, stats, err } if iter.snapshotPinned { @@ -3219,8 +3580,6 @@ func (d *DB) runCompaction( for _, cl := range c.inputs { iter := cl.files.Iter() for f := iter.First(); f != nil; f = iter.Next() { - c.metrics[cl.level].NumFiles-- - c.metrics[cl.level].Size -= int64(f.Size) ve.DeletedFiles[deletedFileEntry{ Level: cl.level, FileNum: f.FileNum, @@ -3228,6 +3587,11 @@ func (d *DB) runCompaction( } } + // The compaction iterator keeps track of a count of the number of DELSIZED + // keys that encoded an incorrect size. Propagate it up as a part of + // compactStats. + stats.countMissizedDels = iter.stats.countMissizedDels + if err := d.objProvider.Sync(); err != nil { return nil, pendingOutputs, stats, err } @@ -3287,7 +3651,7 @@ func (d *DB) scanObsoleteFiles(list []string) { d.mu.compact.cond.Wait() } - liveFileNums := make(map[FileNum]struct{}) + liveFileNums := make(map[base.DiskFileNum]struct{}) d.mu.versions.addLiveFileNums(liveFileNums) // Protect against files which are only referred to by the ingestedFlushable // from being deleted. These are added to the flushable queue on WAL replay @@ -3297,7 +3661,7 @@ func (d *DB) scanObsoleteFiles(list []string) { for _, fEntry := range d.mu.mem.queue { if f, ok := fEntry.flushable.(*ingestedFlushable); ok { for _, file := range f.files { - liveFileNums[file.FileNum] = struct{}{} + liveFileNums[file.FileBacking.DiskFileNum] = struct{}{} } } } @@ -3306,39 +3670,39 @@ func (d *DB) scanObsoleteFiles(list []string) { manifestFileNum := d.mu.versions.manifestFileNum var obsoleteLogs []fileInfo - var obsoleteTables []*fileMetadata + var obsoleteTables []fileInfo var obsoleteManifests []fileInfo var obsoleteOptions []fileInfo for _, filename := range list { - fileType, fileNum, ok := base.ParseFilename(d.opts.FS, filename) + fileType, diskFileNum, ok := base.ParseFilename(d.opts.FS, filename) if !ok { continue } switch fileType { case fileTypeLog: - if fileNum >= minUnflushedLogNum { + if diskFileNum.FileNum() >= minUnflushedLogNum { continue } - fi := fileInfo{fileNum: fileNum} + fi := fileInfo{fileNum: diskFileNum} if stat, err := d.opts.FS.Stat(filename); err == nil { fi.fileSize = uint64(stat.Size()) } obsoleteLogs = append(obsoleteLogs, fi) case fileTypeManifest: - if fileNum >= manifestFileNum { + if diskFileNum.FileNum() >= manifestFileNum { continue } - fi := fileInfo{fileNum: fileNum} + fi := fileInfo{fileNum: diskFileNum} if stat, err := d.opts.FS.Stat(filename); err == nil { fi.fileSize = uint64(stat.Size()) } obsoleteManifests = append(obsoleteManifests, fi) case fileTypeOptions: - if fileNum >= d.optionsFileNum { + if diskFileNum.FileNum() >= d.optionsFileNum.FileNum() { continue } - fi := fileInfo{fileNum: fileNum} + fi := fileInfo{fileNum: diskFileNum} if stat, err := d.opts.FS.Stat(filename); err == nil { fi.fileSize = uint64(stat.Size()) } @@ -3354,16 +3718,16 @@ func (d *DB) scanObsoleteFiles(list []string) { for _, obj := range objects { switch obj.FileType { case fileTypeTable: - if _, ok := liveFileNums[obj.FileNum]; ok { + if _, ok := liveFileNums[obj.DiskFileNum]; ok { continue } - fileMeta := &fileMetadata{ - FileNum: obj.FileNum, + fileInfo := fileInfo{ + fileNum: obj.DiskFileNum, } if size, err := d.objProvider.Size(obj); err == nil { - fileMeta.Size = uint64(size) + fileInfo.fileSize = uint64(size) } - obsoleteTables = append(obsoleteTables, fileMeta) + obsoleteTables = append(obsoleteTables, fileInfo) default: // Ignore object types we don't know about. @@ -3372,7 +3736,7 @@ func (d *DB) scanObsoleteFiles(list []string) { d.mu.log.queue = merge(d.mu.log.queue, obsoleteLogs) d.mu.versions.metrics.WAL.Files = int64(len(d.mu.log.queue)) - d.mu.versions.obsoleteTables = mergeFileMetas(d.mu.versions.obsoleteTables, obsoleteTables) + d.mu.versions.obsoleteTables = mergeFileInfo(d.mu.versions.obsoleteTables, obsoleteTables) d.mu.versions.updateObsoleteTableMetricsLocked() d.mu.versions.obsoleteManifests = merge(d.mu.versions.obsoleteManifests, obsoleteManifests) d.mu.versions.obsoleteOptions = merge(d.mu.versions.obsoleteOptions, obsoleteOptions) @@ -3387,92 +3751,44 @@ func (d *DB) scanObsoleteFiles(list []string) { // // d.mu must be held when calling this method. func (d *DB) disableFileDeletions() { - d.mu.cleaner.disabled++ - for d.mu.cleaner.cleaning { - d.mu.cleaner.cond.Wait() - } - d.mu.cleaner.cond.Broadcast() + d.mu.disableFileDeletions++ + d.mu.Unlock() + defer d.mu.Lock() + d.cleanupManager.Wait() } -// enableFileDeletions enables previously disabled file deletions. Note that if -// file deletions have been re-enabled, the current goroutine will be used to -// perform the queued up deletions. +// enableFileDeletions enables previously disabled file deletions. A cleanup job +// is queued if necessary. // // d.mu must be held when calling this method. func (d *DB) enableFileDeletions() { - if d.mu.cleaner.disabled <= 0 || d.mu.cleaner.cleaning { + if d.mu.disableFileDeletions <= 0 { panic("pebble: file deletion disablement invariant violated") } - d.mu.cleaner.disabled-- - if d.mu.cleaner.disabled > 0 { + d.mu.disableFileDeletions-- + if d.mu.disableFileDeletions > 0 { return } jobID := d.mu.nextJobID d.mu.nextJobID++ - d.deleteObsoleteFiles(jobID, true /* waitForOngoing */) -} - -// d.mu must be held when calling this. -func (d *DB) acquireCleaningTurn(waitForOngoing bool) bool { - // Only allow a single delete obsolete files job to run at a time. - for d.mu.cleaner.cleaning && d.mu.cleaner.disabled == 0 && waitForOngoing { - d.mu.cleaner.cond.Wait() - } - if d.mu.cleaner.cleaning { - return false - } - if d.mu.cleaner.disabled > 0 { - // File deletions are currently disabled. When they are re-enabled a new - // job will be created to catch up on file deletions. - return false - } - d.mu.cleaner.cleaning = true - return true -} - -// d.mu must be held when calling this. -func (d *DB) releaseCleaningTurn() { - d.mu.cleaner.cleaning = false - d.mu.cleaner.cond.Broadcast() -} - -// deleteObsoleteFiles deletes those files and objects that are no longer -// needed. If waitForOngoing is true, it waits for any ongoing cleaning turns to -// complete, and if false, it returns rightaway if a cleaning turn is ongoing. -// -// d.mu must be held when calling this, but the mutex may be dropped and -// re-acquired during the course of this method. -func (d *DB) deleteObsoleteFiles(jobID int, waitForOngoing bool) { - if !d.acquireCleaningTurn(waitForOngoing) { - return - } - d.doDeleteObsoleteFiles(jobID) - d.releaseCleaningTurn() -} - -// obsoleteFile holds information about a file that needs to be deleted soon. -type obsoleteFile struct { - dir string - fileNum base.FileNum - fileType fileType - fileSize uint64 + d.deleteObsoleteFiles(jobID) } type fileInfo struct { - fileNum FileNum + fileNum base.DiskFileNum fileSize uint64 } -// d.mu must be held when calling this, but the mutex may be dropped and -// re-acquired during the course of this method. -func (d *DB) doDeleteObsoleteFiles(jobID int) { - var obsoleteTables []fileInfo - - defer func() { - for _, tbl := range obsoleteTables { - delete(d.mu.versions.zombieTables, tbl.fileNum) - } - }() +// deleteObsoleteFiles enqueues a cleanup job to the cleanup manager, if necessary. +// +// d.mu must be held when calling this. The function will release and re-aquire the mutex. +// +// Does nothing if file deletions are disabled (see disableFileDeletions). A +// cleanup job will be scheduled when file deletions are re-enabled. +func (d *DB) deleteObsoleteFiles(jobID int) { + if d.mu.disableFileDeletions > 0 { + return + } var obsoleteLogs []fileInfo for i := range d.mu.log.queue { @@ -3480,7 +3796,7 @@ func (d *DB) doDeleteObsoleteFiles(jobID int) { // log that has not had its contents flushed to an sstable. We can recycle // the prefix of d.mu.log.queue with log numbers less than // minUnflushedLogNum. - if d.mu.log.queue[i].fileNum >= d.mu.versions.minUnflushedLogNum { + if d.mu.log.queue[i].fileNum.FileNum() >= d.mu.versions.minUnflushedLogNum { obsoleteLogs = d.mu.log.queue[:i] d.mu.log.queue = d.mu.log.queue[i:] d.mu.versions.metrics.WAL.Files -= int64(len(obsoleteLogs)) @@ -3488,19 +3804,18 @@ func (d *DB) doDeleteObsoleteFiles(jobID int) { } } - for _, table := range d.mu.versions.obsoleteTables { - obsoleteTables = append(obsoleteTables, fileInfo{ - fileNum: table.FileNum, - fileSize: table.Size, - }) - } + obsoleteTables := append([]fileInfo(nil), d.mu.versions.obsoleteTables...) d.mu.versions.obsoleteTables = nil + for _, tbl := range obsoleteTables { + delete(d.mu.versions.zombieTables, tbl.fileNum) + } + // Sort the manifests cause we want to delete some contiguous prefix // of the older manifests. sort.Slice(d.mu.versions.obsoleteManifests, func(i, j int) bool { - return d.mu.versions.obsoleteManifests[i].fileNum < - d.mu.versions.obsoleteManifests[j].fileNum + return d.mu.versions.obsoleteManifests[i].fileNum.FileNum() < + d.mu.versions.obsoleteManifests[j].fileNum.FileNum() }) var obsoleteManifests []fileInfo @@ -3516,7 +3831,7 @@ func (d *DB) doDeleteObsoleteFiles(jobID int) { obsoleteOptions := d.mu.versions.obsoleteOptions d.mu.versions.obsoleteOptions = nil - // Release d.mu while doing I/O + // Release d.mu while preparing the cleanup job and possibly waiting. // Note the unusual order: Unlock and then Lock. d.mu.Unlock() defer d.mu.Lock() @@ -3531,12 +3846,12 @@ func (d *DB) doDeleteObsoleteFiles(jobID int) { {fileTypeOptions, obsoleteOptions}, } _, noRecycle := d.opts.Cleaner.(base.NeedsFileContents) - filesToDelete := make([]obsoleteFile, 0, len(files)) + filesToDelete := make([]obsoleteFile, 0, len(obsoleteLogs)+len(obsoleteTables)+len(obsoleteManifests)+len(obsoleteOptions)) for _, f := range files { // We sort to make the order of deletions deterministic, which is nice for // tests. sort.Slice(f.obsolete, func(i, j int) bool { - return f.obsolete[i].fileNum < f.obsolete[j].fileNum + return f.obsolete[i].fileNum.FileNum() < f.obsolete[j].fileNum.FileNum() }) for _, fi := range f.obsolete { dir := d.dirname @@ -3559,121 +3874,24 @@ func (d *DB) doDeleteObsoleteFiles(jobID int) { } } if len(filesToDelete) > 0 { - var tablesSize uint64 - for i := range filesToDelete { - if filesToDelete[i].fileType == fileTypeTable { - tablesSize += filesToDelete[i].fileSize - } - } - d.deletionPacer.reportDeletion(tablesSize) - d.deleters.Add(1) - // Delete asynchronously if that could get held up in the pacer. - if d.opts.Experimental.MinDeletionRate > 0 { - go d.paceAndDeleteObsoleteFiles(jobID, filesToDelete) - } else { - d.paceAndDeleteObsoleteFiles(jobID, filesToDelete) - } + d.cleanupManager.EnqueueJob(jobID, filesToDelete) } -} - -// Paces and eventually deletes the list of obsolete files passed in. db.mu -// must NOT be held when calling this method. -func (d *DB) paceAndDeleteObsoleteFiles(jobID int, files []obsoleteFile) { - defer d.deleters.Done() - - for _, of := range files { - path := base.MakeFilepath(d.opts.FS, of.dir, of.fileType, of.fileNum) - if of.fileType == fileTypeTable { - _ = d.deletionPacer.maybeThrottle(of.fileSize) - d.mu.Lock() - d.mu.versions.metrics.Table.ObsoleteCount-- - d.mu.versions.metrics.Table.ObsoleteSize -= of.fileSize - d.mu.Unlock() - d.deleteObsoleteObject(fileTypeTable, jobID, of.fileNum) - } else { - d.deleteObsoleteFile(of.fileType, jobID, path, of.fileNum) - } + if d.opts.private.testingAlwaysWaitForCleanup { + d.cleanupManager.Wait() } } func (d *DB) maybeScheduleObsoleteTableDeletion() { d.mu.Lock() defer d.mu.Unlock() - - if len(d.mu.versions.obsoleteTables) == 0 { - return - } - if !d.acquireCleaningTurn(false) { - return - } - - go func() { - pprof.Do(context.Background(), gcLabels, func(context.Context) { - d.mu.Lock() - defer d.mu.Unlock() - - jobID := d.mu.nextJobID - d.mu.nextJobID++ - d.doDeleteObsoleteFiles(jobID) - d.releaseCleaningTurn() - }) - }() + d.maybeScheduleObsoleteTableDeletionLocked() } -func (d *DB) deleteObsoleteObject(fileType fileType, jobID int, fileNum FileNum) { - if fileType != fileTypeTable { - panic("not an object") - } - - var path string - meta, err := d.objProvider.Lookup(fileType, fileNum) - if err != nil { - path = "" - } else { - path = d.objProvider.Path(meta) - err = d.objProvider.Remove(fileType, fileNum) - } - if objstorage.IsNotExistError(err) { - return - } - - switch fileType { - case fileTypeTable: - d.opts.EventListener.TableDeleted(TableDeleteInfo{ - JobID: jobID, - Path: path, - FileNum: fileNum, - Err: err, - }) - } -} - -// deleteObsoleteFile deletes a (non-object) file that is no longer needed. -func (d *DB) deleteObsoleteFile(fileType fileType, jobID int, path string, fileNum FileNum) { - // TODO(peter): need to handle this error, probably by re-adding the - // file that couldn't be deleted to one of the obsolete slices map. - err := d.opts.Cleaner.Clean(d.opts.FS, fileType, path) - if oserror.IsNotExist(err) { - return - } - - switch fileType { - case fileTypeLog: - d.opts.EventListener.WALDeleted(WALDeleteInfo{ - JobID: jobID, - Path: path, - FileNum: fileNum, - Err: err, - }) - case fileTypeManifest: - d.opts.EventListener.ManifestDeleted(ManifestDeleteInfo{ - JobID: jobID, - Path: path, - FileNum: fileNum, - Err: err, - }) - case fileTypeTable: - panic("invalid deletion of object file") +func (d *DB) maybeScheduleObsoleteTableDeletionLocked() { + if len(d.mu.versions.obsoleteTables) > 0 { + jobID := d.mu.nextJobID + d.mu.nextJobID++ + d.deleteObsoleteFiles(jobID) } } @@ -3684,7 +3902,7 @@ func merge(a, b []fileInfo) []fileInfo { a = append(a, b...) sort.Slice(a, func(i, j int) bool { - return a[i].fileNum < a[j].fileNum + return a[i].fileNum.FileNum() < a[j].fileNum.FileNum() }) n := 0 @@ -3697,22 +3915,29 @@ func merge(a, b []fileInfo) []fileInfo { return a[:n] } -func mergeFileMetas(a, b []*fileMetadata) []*fileMetadata { +func mergeFileInfo(a, b []fileInfo) []fileInfo { if len(b) == 0 { return a } a = append(a, b...) sort.Slice(a, func(i, j int) bool { - return a[i].FileNum < a[j].FileNum + return a[i].fileNum.FileNum() < a[j].fileNum.FileNum() }) n := 0 for i := 0; i < len(a); i++ { - if n == 0 || a[i].FileNum != a[n-1].FileNum { + if n == 0 || a[i].fileNum != a[n-1].fileNum { a[n] = a[i] n++ } } return a[:n] } + +func max[I constraints.Ordered](a, b I) I { + if b > a { + return b + } + return a +} diff --git a/vendor/github.com/cockroachdb/pebble/compaction_iter.go b/vendor/github.com/cockroachdb/pebble/compaction_iter.go index 8434fa6..299dbfc 100644 --- a/vendor/github.com/cockroachdb/pebble/compaction_iter.go +++ b/vendor/github.com/cockroachdb/pebble/compaction_iter.go @@ -6,6 +6,7 @@ package pebble import ( "bytes" + "encoding/binary" "fmt" "io" "sort" @@ -14,9 +15,9 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/bytealloc" - "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/rangekey" + "github.com/cockroachdb/redact" ) // compactionIter provides a forward-only iterator that encapsulates the logic @@ -205,7 +206,29 @@ type compactionIter struct { // compaction iterator was only returned because an open snapshot prevents // its elision. This field only applies to point keys, and not to range // deletions or range keys. + // + // For MERGE, it is possible that doing the merge is interrupted even when + // the next point key is in the same stripe. This can happen if the loop in + // mergeNext gets interrupted by sameStripeNonSkippable. + // sameStripeNonSkippable occurs due to RANGEDELs that sort before + // SET/MERGE/DEL with the same seqnum, so the RANGEDEL does not necessarily + // delete the subsequent SET/MERGE/DEL keys. snapshotPinned bool + // forceObsoleteDueToRangeDel is set to true in a subset of the cases that + // snapshotPinned is true. This value is true when the point is obsolete due + // to a RANGEDEL but could not be deleted due to a snapshot. + // + // NB: it may seem that the additional cases that snapshotPinned captures + // are harmless in that they can also be used to mark a point as obsolete + // (it is merely a duplication of some logic that happens in + // Writer.AddWithForceObsolete), but that is not quite accurate as of this + // writing -- snapshotPinned originated in stats collection and for a + // sequence MERGE, SET, where the MERGE cannot merge with the (older) SET + // due to a snapshot, the snapshotPinned value for the SET is true. + // + // TODO(sumeer,jackson): improve the logic of snapshotPinned and reconsider + // whether we need forceObsoleteDueToRangeDel. + forceObsoleteDueToRangeDel bool // The index of the snapshot for the current key within the snapshots slice. curSnapshotIdx int curSnapshotSeqNum uint64 @@ -233,13 +256,19 @@ type compactionIter struct { // The fragmented range keys. rangeKeys []keyspan.Span // Byte allocator for the tombstone keys. - alloc bytealloc.A - allowZeroSeqNum bool - elideTombstone func(key []byte) bool - elideRangeTombstone func(start, end []byte) bool + alloc bytealloc.A + allowZeroSeqNum bool + elideTombstone func(key []byte) bool + elideRangeTombstone func(start, end []byte) bool + ineffectualSingleDeleteCallback func(userKey []byte) + singleDeleteInvariantViolationCallback func(userKey []byte) // The on-disk format major version. This informs the types of keys that // may be written to disk during a compaction. formatVersion FormatMajorVersion + stats struct { + // count of DELSIZED keys that were missized. + countMissizedDels uint64 + } } func newCompactionIter( @@ -254,20 +283,24 @@ func newCompactionIter( allowZeroSeqNum bool, elideTombstone func(key []byte) bool, elideRangeTombstone func(start, end []byte) bool, + ineffectualSingleDeleteCallback func(userKey []byte), + singleDeleteInvariantViolationCallback func(userKey []byte), formatVersion FormatMajorVersion, ) *compactionIter { i := &compactionIter{ - equal: equal, - merge: merge, - iter: iter, - snapshots: snapshots, - frontiers: frontiers{cmp: cmp}, - rangeDelFrag: rangeDelFrag, - rangeKeyFrag: rangeKeyFrag, - allowZeroSeqNum: allowZeroSeqNum, - elideTombstone: elideTombstone, - elideRangeTombstone: elideRangeTombstone, - formatVersion: formatVersion, + equal: equal, + merge: merge, + iter: iter, + snapshots: snapshots, + frontiers: frontiers{cmp: cmp}, + rangeDelFrag: rangeDelFrag, + rangeKeyFrag: rangeKeyFrag, + allowZeroSeqNum: allowZeroSeqNum, + elideTombstone: elideTombstone, + elideRangeTombstone: elideRangeTombstone, + ineffectualSingleDeleteCallback: ineffectualSingleDeleteCallback, + singleDeleteInvariantViolationCallback: singleDeleteInvariantViolationCallback, + formatVersion: formatVersion, } i.rangeDelFrag.Cmp = cmp i.rangeDelFrag.Format = formatKey @@ -306,21 +339,43 @@ func (i *compactionIter) Next() (*InternalKey, []byte) { return nil, nil } - // Prior to this call to `Next()` we are in one of three situations with + // Prior to this call to `Next()` we are in one of four situations with // respect to `iterKey` and related state: // // - `!skip && pos == iterPosNext`: `iterKey` is already at the next key. - // - `!skip && pos == iterPosCur`: We are at the key that has been returned. + // - `!skip && pos == iterPosCurForward`: We are at the key that has been returned. // To move forward we advance by one key, even if that lands us in the same // snapshot stripe. - // - `skip && pos == iterPosCur`: We are at the key that has been returned. + // - `skip && pos == iterPosCurForward`: We are at the key that has been returned. // To move forward we skip skippable entries in the stripe. + // - `skip && pos == iterPosNext && i.iterStripeChange == sameStripeNonSkippable`: + // This case may occur when skipping within a snapshot stripe and we + // encounter either: + // a) an invalid key kind; The previous call will have returned + // whatever key it was processing and deferred handling of the + // invalid key to this invocation of Next(). We're responsible for + // ignoring skip=true and falling into the invalid key kind case + // down below. + // b) an interleaved range delete; This is a wart of the current code + // structure. While skipping within a snapshot stripe, a range + // delete interleaved at its start key and sequence number + // interrupts the sequence of point keys. After we return the range + // delete to the caller, we need to pick up skipping at where we + // left off, so we preserve skip=true. + // TODO(jackson): This last case is confusing and can be removed if we + // interleave range deletions at the maximal sequence number using the + // keyspan interleaving iterator. This is the treatment given to range + // keys today. if i.pos == iterPosCurForward { if i.skip { i.skipInStripe() } else { i.nextInStripe() } + } else if i.skip { + if i.iterStripeChange != sameStripeNonSkippable { + panic(errors.AssertionFailedf("compaction iterator has skip=true, but iterator is at iterPosNext")) + } } i.pos = iterPosCurForward @@ -374,6 +429,10 @@ func (i *compactionIter) Next() (*InternalKey, []byte) { return &i.key, i.value } + // TODO(sumeer): we could avoid calling Covers if i.iterStripeChange == + // sameStripeSameKey since that check has already been done in + // nextInStripeHelper. However, we also need to handle the case of + // CoversInvisibly below. if cover := i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum); cover == keyspan.CoversVisibly { // A pending range deletion deletes this key. Skip it. i.saveKey() @@ -382,17 +441,37 @@ func (i *compactionIter) Next() (*InternalKey, []byte) { } else if cover == keyspan.CoversInvisibly { // i.iterKey would be deleted by a range deletion if there weren't // any open snapshots. Mark it as pinned. + // + // NB: there are multiple places in this file where we call + // i.rangeDelFrag.Covers and this is the only one where we are writing + // to i.snapshotPinned. Those other cases occur in mergeNext where the + // caller is deciding whether the value should be merged or not, and the + // key is in the same snapshot stripe. Hence, snapshotPinned is by + // definition false in those cases. i.snapshotPinned = true + i.forceObsoleteDueToRangeDel = true + } else { + i.forceObsoleteDueToRangeDel = false } switch i.iterKey.Kind() { - case InternalKeyKindDelete, InternalKeyKindSingleDelete: + case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized: if i.elideTombstone(i.iterKey.UserKey) { if i.curSnapshotIdx == 0 { // If we're at the last snapshot stripe and the tombstone // can be elided skip skippable keys in the same stripe. i.saveKey() - i.skipInStripe() + if i.key.Kind() == InternalKeyKindSingleDelete { + i.skipDueToSingleDeleteElision() + } else { + i.skipInStripe() + if !i.skip && i.iterStripeChange != newStripeNewKey { + panic(errors.AssertionFailedf("pebble: skipInStripe in last stripe disabled skip without advancing to new key")) + } + } + if i.iterStripeChange == newStripeSameKey { + panic(errors.AssertionFailedf("pebble: skipInStripe in last stripe found a new stripe within the same key")) + } continue } else { // We're not at the last snapshot stripe, so the tombstone @@ -410,11 +489,23 @@ func (i *compactionIter) Next() (*InternalKey, []byte) { i.skip = true return &i.key, i.value + case InternalKeyKindDeleteSized: + // We may skip subsequent keys because of this tombstone. Scan + // ahead to see just how much data this tombstone drops and if + // the tombstone's value should be updated accordingly. + return i.deleteSizedNext() + case InternalKeyKindSingleDelete: if i.singleDeleteNext() { return &i.key, i.value + } else if i.err != nil { + return nil, nil } continue + + default: + panic(errors.AssertionFailedf( + "unexpected kind %s", redact.SafeString(i.iterKey.Kind().String()))) } case InternalKeyKindSet, InternalKeyKindSetWithDelete: @@ -424,6 +515,9 @@ func (i *compactionIter) Next() (*InternalKey, []byte) { // preserving the original value, and potentially mutating the key // kind. i.setNext() + if i.err != nil { + return nil, nil + } return &i.key, i.value case InternalKeyKindMerge: @@ -440,7 +534,15 @@ func (i *compactionIter) Next() (*InternalKey, []byte) { if i.err == nil { // includesBase is true whenever we've transformed the MERGE record // into a SET. - includesBase := i.key.Kind() == InternalKeyKindSet + var includesBase bool + switch i.key.Kind() { + case InternalKeyKindSet, InternalKeyKindSetWithDelete: + includesBase = true + case InternalKeyKindMerge: + default: + panic(errors.AssertionFailedf( + "unexpected kind %s", redact.SafeString(i.key.Kind().String()))) + } i.value, needDelete, i.valueCloser, i.err = finishValueMerger(valueMerger, includesBase) } if i.err == nil { @@ -469,6 +571,8 @@ func (i *compactionIter) Next() (*InternalKey, []byte) { } if i.err != nil { i.valid = false + // TODO(sumeer): why is MarkCorruptionError only being called for + // MERGE? i.err = base.MarkCorruptionError(i.err) } return nil, nil @@ -508,10 +612,16 @@ func snapshotIndex(seq uint64, snapshots []uint64) (int, uint64) { return index, snapshots[index] } -// skipInStripe skips over skippable keys in the same stripe and user key. +// skipInStripe skips over skippable keys in the same stripe and user key. It +// may set i.err, in which case i.iterKey will be nil. func (i *compactionIter) skipInStripe() { i.skip = true + // TODO(sumeer): we can avoid the overhead of calling i.rangeDelFrag.Covers, + // in this case of nextInStripe, since we are skipping all of them anyway. for i.nextInStripe() == sameStripeSkippable { + if i.err != nil { + panic(i.err) + } } // Reset skip if we landed outside the original stripe. Otherwise, we landed // in the same stripe on a non-skippable key. In that case we should preserve @@ -549,11 +659,17 @@ const ( // nextInStripe advances the iterator and returns one of the above const ints // indicating how its state changed. // +// All sameStripeSkippable keys that are covered by a RANGEDEL will be skipped +// and not returned. +// // Calls to nextInStripe must be preceded by a call to saveKey to retain a // temporary reference to the original key, so that forward iteration can // proceed with a reference to the original key. Care should be taken to avoid // overwriting or mutating the saved key or value before they have been returned // to the caller of the exported function (i.e. the caller of Next, First, etc.) +// +// nextInStripe may set i.err, in which case the return value will be +// newStripeNewKey, and i.iterKey will be nil. func (i *compactionIter) nextInStripe() stripeChangeType { i.iterStripeChange = i.nextInStripeHelper() return i.iterStripeChange @@ -562,58 +678,72 @@ func (i *compactionIter) nextInStripe() stripeChangeType { // nextInStripeHelper is an internal helper for nextInStripe; callers should use // nextInStripe and not call nextInStripeHelper. func (i *compactionIter) nextInStripeHelper() stripeChangeType { - if !i.iterNext() { - return newStripeNewKey - } - key := i.iterKey + origSnapshotIdx := i.curSnapshotIdx + for { + if !i.iterNext() { + return newStripeNewKey + } + key := i.iterKey - // NB: The below conditional is an optimization to avoid a user key - // comparison in many cases. Internal keys with the same user key are - // ordered in (strictly) descending order by trailer. If the new key has a - // greater or equal trailer, or the previous key had a zero sequence number, - // the new key must have a new user key. - // - // A couple things make these cases common: - // - Sequence-number zeroing ensures ~all of the keys in L6 have a zero - // sequence number. - // - Ingested sstables' keys all adopt the same sequence number. - if i.keyTrailer <= base.InternalKeyZeroSeqnumMaxTrailer || key.Trailer >= i.keyTrailer { - if invariants.Enabled && i.equal(i.key.UserKey, key.UserKey) { + if !i.equal(i.key.UserKey, key.UserKey) { + i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots) + return newStripeNewKey + } + + // If i.key and key have the same user key, then + // 1. i.key must not have had a zero sequence number (or it would've be the last + // key with its user key). + // 2. i.key must have a strictly larger sequence number + // There's an exception in that either key may be a range delete. Range + // deletes may share a sequence number with a point key if the keys were + // ingested together. Range keys may also share the sequence number if they + // were ingested, but range keys are interleaved into the compaction + // iterator's input iterator at the maximal sequence number so their + // original sequence number will not be observed here. + if prevSeqNum := base.SeqNumFromTrailer(i.keyTrailer); (prevSeqNum == 0 || prevSeqNum <= key.SeqNum()) && + i.key.Kind() != InternalKeyKindRangeDelete && key.Kind() != InternalKeyKindRangeDelete { prevKey := i.key prevKey.Trailer = i.keyTrailer - panic(fmt.Sprintf("pebble: invariant violation: %s and %s out of order", key, prevKey)) + panic(errors.AssertionFailedf("pebble: invariant violation: %s and %s out of order", prevKey, key)) } + i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots) - return newStripeNewKey - } else if !i.equal(i.key.UserKey, key.UserKey) { - i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots) - return newStripeNewKey - } - origSnapshotIdx := i.curSnapshotIdx - i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots) - switch key.Kind() { - case InternalKeyKindRangeDelete: - // Range tombstones need to be exposed by the compactionIter to the upper level - // `compaction` object, so return them regardless of whether they are in the same - // snapshot stripe. - if i.curSnapshotIdx == origSnapshotIdx { - return sameStripeNonSkippable + switch key.Kind() { + case InternalKeyKindRangeDelete: + // Range tombstones need to be exposed by the compactionIter to the upper level + // `compaction` object, so return them regardless of whether they are in the same + // snapshot stripe. + if i.curSnapshotIdx == origSnapshotIdx { + return sameStripeNonSkippable + } + return newStripeSameKey + case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: + // Range keys are interleaved at the max sequence number for a given user + // key, so we should not see any more range keys in this stripe. + panic("unreachable") + case InternalKeyKindInvalid: + if i.curSnapshotIdx == origSnapshotIdx { + return sameStripeNonSkippable + } + return newStripeSameKey + case InternalKeyKindDelete, InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSingleDelete, + InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized: + // Fall through + default: + i.iterKey = nil + i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind())) + i.valid = false + return newStripeNewKey } - return newStripeSameKey - case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: - // Range keys are interleaved at the max sequence number for a given user - // key, so we should not see any more range keys in this stripe. - panic("unreachable") - case InternalKeyKindInvalid: if i.curSnapshotIdx == origSnapshotIdx { - return sameStripeNonSkippable + // Same snapshot. + if i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) == keyspan.CoversVisibly { + continue + } + return sameStripeSkippable } return newStripeSameKey } - if i.curSnapshotIdx == origSnapshotIdx { - return sameStripeSkippable - } - return newStripeSameKey } func (i *compactionIter) setNext() { @@ -639,6 +769,10 @@ func (i *compactionIter) setNext() { // Else, we continue to loop through entries in the stripe looking for a // DEL. Note that we may stop *before* encountering a DEL, if one exists. + // + // NB: nextInStripe will skip sameStripeSkippable keys that are visibly + // covered by a RANGEDEL. This can include DELs -- this is fine since such + // DELs don't need to be combined with SET to make SETWITHDEL. for { switch i.nextInStripe() { case newStripeNewKey, newStripeSameKey: @@ -686,14 +820,19 @@ func (i *compactionIter) setNext() { i.skip = true return case sameStripeSkippable: - // We're still in the same stripe. If this is a DEL/SINGLEDEL, we - // stop looking and emit a SETWITHDEL. Subsequent keys are - // eligible for skipping. - if i.iterKey.Kind() == InternalKeyKindDelete || - i.iterKey.Kind() == InternalKeyKindSingleDelete { + // We're still in the same stripe. If this is a + // DEL/SINGLEDEL/DELSIZED, we stop looking and emit a SETWITHDEL. + // Subsequent keys are eligible for skipping. + switch i.iterKey.Kind() { + case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized: i.key.SetKind(InternalKeyKindSetWithDelete) i.skip = true return + case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete: + // Do nothing + default: + i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind())) + i.valid = false } default: panic("pebble: unexpected stripeChangeType: " + strconv.Itoa(int(i.iterStripeChange))) @@ -713,13 +852,24 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType { i.pos = iterPosNext return i.iterStripeChange } + if i.err != nil { + panic(i.err) + } + // NB: MERGE#10+RANGEDEL#9 stays a MERGE, since nextInStripe skips + // sameStripeSkippable keys that are visibly covered by a RANGEDEL. There + // may be MERGE#7 that is invisibly covered and will be preserved, but + // there is no risk that MERGE#10 and MERGE#7 will get merged in the + // future as the RANGEDEL still exists and will be used in user-facing + // reads that see MERGE#10, and will also eventually cause MERGE#7 to be + // deleted in a compaction. key := i.iterKey switch key.Kind() { - case InternalKeyKindDelete, InternalKeyKindSingleDelete: + case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized: // We've hit a deletion tombstone. Return everything up to this point and // then skip entries until the next snapshot stripe. We change the kind // of the result key to a Set so that it shadows keys in lower - // levels. That is, MERGE+DEL -> SET. + // levels. That is, MERGE+DEL -> SETWITHDEL. + // // We do the same for SingleDelete since SingleDelete is only // permitted (with deterministic behavior) for keys that have been // set once since the last SingleDelete/Delete, so everything @@ -732,21 +882,11 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType { // single Set, and then merge in any following Sets, but that is // complicated wrt code and unnecessary given the narrow permitted // use of SingleDelete. - i.key.SetKind(InternalKeyKindSet) + i.key.SetKind(InternalKeyKindSetWithDelete) i.skip = true return sameStripeSkippable case InternalKeyKindSet, InternalKeyKindSetWithDelete: - if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly { - // We change the kind of the result key to a Set so that it shadows - // keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't - // strictly necessary, but provides consistency with the behavior of - // MERGE+DEL. - i.key.SetKind(InternalKeyKindSet) - i.skip = true - return sameStripeSkippable - } - // We've hit a Set or SetWithDel value. Merge with the existing // value and return. We change the kind of the resulting key to a // Set so that it shadows keys in lower levels. That is: @@ -761,16 +901,6 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType { return sameStripeSkippable case InternalKeyKindMerge: - if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly { - // We change the kind of the result key to a Set so that it shadows - // keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't - // strictly necessary, but provides consistency with the behavior of - // MERGE+DEL. - i.key.SetKind(InternalKeyKindSet) - i.skip = true - return sameStripeSkippable - } - // We've hit another Merge value. Merge with the existing value and // continue looping. i.err = valueMerger.MergeOlder(i.iterValue) @@ -787,6 +917,16 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType { } } +// singleDeleteNext processes a SingleDelete point tombstone. A SingleDelete, or +// SINGLEDEL, is unique in that it deletes exactly 1 internal key. It's a +// performance optimization when the client knows a user key has not been +// overwritten, allowing the elision of the tombstone earlier, avoiding write +// amplification. +// +// singleDeleteNext returns a boolean indicating whether or not the caller +// should yield the SingleDelete key to the consumer of the compactionIter. If +// singleDeleteNext returns false, the caller may consume/elide the +// SingleDelete. func (i *compactionIter) singleDeleteNext() bool { // Save the current key. i.saveKey() @@ -795,26 +935,85 @@ func (i *compactionIter) singleDeleteNext() bool { // Loop until finds a key to be passed to the next level. for { + // If we find a key that can't be skipped, return true so that the + // caller yields the SingleDelete to the caller. if i.nextInStripe() != sameStripeSkippable { + // This defers additional error checking regarding single delete + // invariants to the compaction where the keys with the same user key as + // the single delete are in the same stripe. i.pos = iterPosNext - return true + return i.err == nil } - + if i.err != nil { + panic(i.err) + } + // INVARIANT: sameStripeSkippable. key := i.iterKey - switch key.Kind() { - case InternalKeyKindDelete, InternalKeyKindMerge, InternalKeyKindSetWithDelete: - // We've hit a Delete, Merge or SetWithDelete, transform the - // SingleDelete into a full Delete. + kind := key.Kind() + switch kind { + case InternalKeyKindDelete, InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized: + if (kind == InternalKeyKindDelete || kind == InternalKeyKindDeleteSized) && + i.ineffectualSingleDeleteCallback != nil { + i.ineffectualSingleDeleteCallback(i.key.UserKey) + } + // We've hit a Delete, DeleteSized, SetWithDelete, transform + // the SingleDelete into a full Delete. i.key.SetKind(InternalKeyKindDelete) i.skip = true return true - case InternalKeyKindSet: - i.nextInStripe() + case InternalKeyKindSet, InternalKeyKindMerge: + // This SingleDelete deletes the Set/Merge, and we can now elide the + // SingleDel as well. We advance past the Set and return false to + // indicate to the main compaction loop that we should NOT yield the + // current SingleDel key to the compaction loop. + // + // NB: singleDeleteNext was called with i.pos == iterPosCurForward, and + // after the call to nextInStripe, we are still at iterPosCurForward, + // since we are at the key after the Set/Merge that was single deleted. + change := i.nextInStripe() + switch change { + case sameStripeSkippable, newStripeSameKey: + // On the same user key. + nextKind := i.iterKey.Kind() + switch nextKind { + case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge: + if i.singleDeleteInvariantViolationCallback != nil { + // sameStripeSkippable keys returned by nextInStripe() are already + // known to not be covered by a RANGEDEL, so it is an invariant + // violation. The rare case is newStripeSameKey, where it is a + // violation if not covered by a RANGEDEL. + if change == sameStripeSkippable || + i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) == keyspan.NoCover { + i.singleDeleteInvariantViolationCallback(i.key.UserKey) + } + } + case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete, + InternalKeyKindRangeDelete: + default: + panic(errors.AssertionFailedf( + "unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind()))) + } + case sameStripeNonSkippable: + // No ability to check whether there is another Set/Merge below with + // the same user key. + // + // TODO(sumeer): once range deletions are interleaved at the maximal + // sequence number, this case will go away. + case newStripeNewKey: + default: + panic("unreachable") + } i.valid = false return false case InternalKeyKindSingleDelete: + // Two single deletes met in a compaction. The first single delete is + // ineffectual. + if i.ineffectualSingleDeleteCallback != nil { + i.ineffectualSingleDeleteCallback(i.key.UserKey) + } + // Continue to apply the second single delete. continue default: @@ -825,6 +1024,303 @@ func (i *compactionIter) singleDeleteNext() bool { } } +// skipDueToSingleDeleteElision is called when the SingleDelete is being +// elided because it is in the final snapshot stripe and there are no keys +// with the same user key in lower levels in the LSM (below the files in this +// compaction). +// +// TODO(sumeer): the only difference between singleDeleteNext and +// skipDueToSingleDeleteElision is the fact that the caller knows it will be +// eliding the single delete in the latter case. There are some similar things +// happening in both implementations. My first attempt at combining them into +// a single method was hard to comprehend. Try again. +func (i *compactionIter) skipDueToSingleDeleteElision() { + for { + stripeChange := i.nextInStripe() + if i.err != nil { + panic(i.err) + } + switch stripeChange { + case newStripeNewKey: + // The single delete is only now being elided, meaning it did not elide + // any keys earlier in its descent down the LSM. We stepped onto a new + // user key, meaning that even now at its moment of elision, it still + // hasn't elided any other keys. The single delete was ineffectual (a + // no-op). + if i.ineffectualSingleDeleteCallback != nil { + i.ineffectualSingleDeleteCallback(i.key.UserKey) + } + i.skip = false + return + case newStripeSameKey: + // This should be impossible. If we're eliding a single delete, we + // determined that the tombstone is in the final snapshot stripe, but we + // stepped into a new stripe of the same key. + panic(errors.AssertionFailedf("eliding single delete followed by same key in new stripe")) + case sameStripeNonSkippable: + // There's a key that we cannot skip. There are two possible cases: + // a. The key is invalid. This is an error. + // b. The key is a range deletion. + // The second case may also be an ineffectual single delete. However, it + // is possible that there is a SET that is at the same seqnum as the + // RANGEDEL, and so is not deleted by that RANGEDEL, and will be deleted + // by this single delete. So we cannot be certain that this is an + // ineffectual single delete. + // + // TODO(sumeer): the existing todo to interleave range deletions at the + // maximal sequence number will allow us to address this ambiguity. + // + // TODO(sumeer): by setting skip to true, the compactionIter is making a + // single delete stronger (like a del), which will hide bugs in the use of + // single delete. + i.skip = true + return + case sameStripeSkippable: + kind := i.iterKey.Kind() + switch kind { + case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete: + if i.ineffectualSingleDeleteCallback != nil { + i.ineffectualSingleDeleteCallback(i.key.UserKey) + } + switch kind { + case InternalKeyKindDelete, InternalKeyKindDeleteSized: + i.skipInStripe() + return + case InternalKeyKindSingleDelete: + // Repeat the same with this SingleDelete. We don't want to simply + // call skipInStripe(), since it increases the strength of the + // SingleDel, which hides bugs in the use of single delete. + continue + default: + panic(errors.AssertionFailedf( + "unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind()))) + } + case InternalKeyKindSetWithDelete: + // The SingleDelete should behave like a Delete. + i.skipInStripe() + return + case InternalKeyKindSet, InternalKeyKindMerge: + // This SingleDelete deletes the Set/Merge, and we are eliding the + // SingleDel as well. Step to the next key (this is not deleted by the + // SingleDelete). + // + // NB: skipDueToSingleDeleteElision was called with i.pos == + // iterPosCurForward, and after the call to nextInStripe, we are still + // at iterPosCurForward, since we are at the key after the Set/Merge + // that was single deleted. + change := i.nextInStripe() + if i.err != nil { + panic(i.err) + } + switch change { + case newStripeSameKey: + panic(errors.AssertionFailedf("eliding single delete followed by same key in new stripe")) + case newStripeNewKey: + case sameStripeSkippable: + // On the same key. + nextKind := i.iterKey.Kind() + switch nextKind { + case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge: + if i.singleDeleteInvariantViolationCallback != nil { + i.singleDeleteInvariantViolationCallback(i.key.UserKey) + } + case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete, + InternalKeyKindRangeDelete: + default: + panic(errors.AssertionFailedf( + "unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind()))) + } + case sameStripeNonSkippable: + // No ability to check whether there is another Set/Merge below with + // the same user key. + // + // TODO(sumeer): once range deletions are interleaved at the maximal + // sequence number, this case will go away. + default: + panic("unreachable") + } + // Whether in same stripe or new stripe, this key is not consumed by + // the SingleDelete. + i.skip = false + return + default: + panic(errors.AssertionFailedf( + "unexpected internal key kind: %d", errors.Safe(i.iterKey.Kind()))) + } + default: + panic("unreachable") + } + } +} + +// deleteSizedNext processes a DELSIZED point tombstone. Unlike ordinary DELs, +// these tombstones carry a value that's a varint indicating the size of the +// entry (len(key)+len(value)) that the tombstone is expected to delete. +// +// When a deleteSizedNext is encountered, we skip ahead to see which keys, if +// any, are elided as a result of the tombstone. +func (i *compactionIter) deleteSizedNext() (*base.InternalKey, []byte) { + i.saveKey() + i.valid = true + i.skip = true + + // The DELSIZED tombstone may have no value at all. This happens when the + // tombstone has already deleted the key that the user originally predicted. + // In this case, we still peek forward in case there's another DELSIZED key + // with a lower sequence number, in which case we'll adopt its value. + if len(i.iterValue) == 0 { + i.value = i.valueBuf[:0] + } else { + i.valueBuf = append(i.valueBuf[:0], i.iterValue...) + i.value = i.valueBuf + } + + // Loop through all the keys within this stripe that are skippable. + i.pos = iterPosNext + for i.nextInStripe() == sameStripeSkippable { + if i.err != nil { + panic(i.err) + } + switch i.iterKey.Kind() { + case InternalKeyKindDelete, InternalKeyKindDeleteSized, InternalKeyKindSingleDelete: + // We encountered a tombstone (DEL, or DELSIZED) that's deleted by + // the original DELSIZED tombstone. This can happen in two cases: + // + // (1) These tombstones were intended to delete two distinct values, + // and this DELSIZED has already dropped the relevant key. For + // example: + // + // a.DELSIZED.9 a.SET.7 a.DELSIZED.5 a.SET.4 + // + // If a.DELSIZED.9 has already deleted a.SET.7, its size has + // already been zeroed out. In this case, we want to adopt the + // value of the DELSIZED with the lower sequence number, in + // case the a.SET.4 key has not yet been elided. + // + // (2) This DELSIZED was missized. The user thought they were + // deleting a key with this user key, but this user key had + // already been deleted. + // + // We can differentiate these two cases by examining the length of + // the DELSIZED's value. A DELSIZED's value holds the size of both + // the user key and value that it intends to delete. For any user + // key with a length > 0, a DELSIZED that has not deleted a key must + // have a value with a length > 0. + // + // We treat both cases the same functionally, adopting the identity + // of the lower-sequence numbered tombstone. However in the second + // case, we also increment the stat counting missized tombstones. + if len(i.value) > 0 { + // The original DELSIZED key was missized. The key that the user + // thought they were deleting does not exist. + i.stats.countMissizedDels++ + } + i.valueBuf = append(i.valueBuf[:0], i.iterValue...) + i.value = i.valueBuf + if i.iterKey.Kind() != InternalKeyKindDeleteSized { + // Convert the DELSIZED to a DEL—The DEL/SINGLEDEL we're eliding + // may not have deleted the key(s) it was intended to yet. The + // ordinary DEL compaction heuristics are better suited at that, + // plus we don't want to count it as a missized DEL. We early + // exit in this case, after skipping the remainder of the + // snapshot stripe. + i.key.SetKind(InternalKeyKindDelete) + // NB: We skipInStripe now, rather than returning leaving + // i.skip=true and returning early, because Next() requires + // that i.skip=true only if i.iterPos = iterPosCurForward. + // + // Ignore any error caused by skipInStripe since it does not affect + // the key/value being returned here, and the next call to Next() will + // expose it. + i.skipInStripe() + return &i.key, i.value + } + // Continue, in case we uncover another DELSIZED or a key this + // DELSIZED deletes. + + case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete: + // If the DELSIZED is value-less, it already deleted the key that it + // was intended to delete. This is possible with a sequence like: + // + // DELSIZED.8 SET.7 SET.3 + // + // The DELSIZED only describes the size of the SET.7, which in this + // case has already been elided. We don't count it as a missizing, + // instead converting the DELSIZED to a DEL. Skip the remainder of + // the snapshot stripe and return. + if len(i.value) == 0 { + i.key.SetKind(InternalKeyKindDelete) + // NB: We skipInStripe now, rather than returning leaving + // i.skip=true and returning early, because Next() requires + // that i.skip=true only if i.iterPos = iterPosCurForward. + // + // Ignore any error caused by skipInStripe since it does not affect + // the key/value being returned here, and the next call to Next() will + // expose it. + i.skipInStripe() + return &i.key, i.value + } + // The deleted key is not a DEL, DELSIZED, and the DELSIZED in i.key + // has a positive size. + expectedSize, n := binary.Uvarint(i.value) + if n != len(i.value) { + i.err = base.CorruptionErrorf("DELSIZED holds invalid value: %x", errors.Safe(i.value)) + i.valid = false + return nil, nil + } + elidedSize := uint64(len(i.iterKey.UserKey)) + uint64(len(i.iterValue)) + if elidedSize != expectedSize { + // The original DELSIZED key was missized. It's unclear what to + // do. The user-provided size was wrong, so it's unlikely to be + // accurate or meaningful. We could: + // + // 1. return the DELSIZED with the original user-provided size unmodified + // 2. return the DELZIZED with a zeroed size to reflect that a key was + // elided, even if it wasn't the anticipated size. + // 3. subtract the elided size from the estimate and re-encode. + // 4. convert the DELSIZED into a value-less DEL, so that + // ordinary DEL heuristics apply. + // + // We opt for (4) under the rationale that we can't rely on the + // user-provided size for accuracy, so ordinary DEL heuristics + // are safer. + i.stats.countMissizedDels++ + i.key.SetKind(InternalKeyKindDelete) + i.value = i.valueBuf[:0] + // NB: We skipInStripe now, rather than returning leaving + // i.skip=true and returning early, because Next() requires + // that i.skip=true only if i.iterPos = iterPosCurForward. + // + // Ignore any error caused by skipInStripe since it does not affect + // the key/value being returned here, and the next call to Next() will + // expose it. + i.skipInStripe() + return &i.key, i.value + } + // NB: We remove the value regardless of whether the key was sized + // appropriately. The size encoded is 'consumed' the first time it + // meets a key that it deletes. + i.value = i.valueBuf[:0] + + default: + i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind())) + i.valid = false + return nil, nil + } + } + // Reset skip if we landed outside the original stripe. Otherwise, we landed + // in the same stripe on a non-skippable key. In that case we should preserve + // `i.skip == true` such that later keys in the stripe will continue to be + // skipped. + if i.iterStripeChange == newStripeNewKey || i.iterStripeChange == newStripeSameKey { + i.skip = false + } + if i.err != nil { + return nil, nil + } + return &i.key, i.value +} + func (i *compactionIter) saveKey() { i.keyBuf = append(i.keyBuf[:0], i.iterKey.UserKey...) i.key.UserKey = i.keyBuf @@ -957,7 +1453,7 @@ func (i *compactionIter) maybeZeroSeqnum(snapshotIdx int) { // This is not the last snapshot return } - i.key.SetSeqNum(0) + i.key.SetSeqNum(base.SeqNumZero) } // A frontier is used to monitor a compaction's progression across the user diff --git a/vendor/github.com/cockroachdb/pebble/compaction_picker.go b/vendor/github.com/cockroachdb/pebble/compaction_picker.go index afd15b0..f35d475 100644 --- a/vendor/github.com/cockroachdb/pebble/compaction_picker.go +++ b/vendor/github.com/cockroachdb/pebble/compaction_picker.go @@ -21,6 +21,17 @@ import ( const minIntraL0Count = 4 type compactionEnv struct { + // diskAvailBytes holds a statistic on the number of bytes available on + // disk, as reported by the filesystem. It's used to be more restrictive in + // expanding compactions if available disk space is limited. + // + // The cached value (d.diskAvailBytes) is updated whenever a file is deleted + // and whenever a compaction or flush completes. Since file removal is the + // primary means of reclaiming space, there is a rough bound on the + // statistic's staleness when available bytes is growing. Compactions and + // flushes are longer, slower operations and provide a much looser bound + // when available bytes is decreasing. + diskAvailBytes uint64 earliestUnflushedSeqNum uint64 earliestSnapshotSeqNum uint64 inProgressCompactions []compactionInfo @@ -30,10 +41,8 @@ type compactionEnv struct { type compactionPicker interface { getScores([]compactionInfo) [numLevels]float64 getBaseLevel() int - getEstimatedMaxWAmp() float64 estimatedCompactionDebt(l0ExtraSize uint64) uint64 pickAuto(env compactionEnv) (pc *pickedCompaction) - pickManual(env compactionEnv, manual *manualCompaction) (c *pickedCompaction, retryLater bool) pickElisionOnlyCompaction(env compactionEnv) (pc *pickedCompaction) pickRewriteCompaction(env compactionEnv) (pc *pickedCompaction) pickReadTriggeredCompaction(env compactionEnv) (pc *pickedCompaction) @@ -47,13 +56,17 @@ type readCompactionEnv struct { flushing bool } -// Information about in-progress compactions provided to the compaction picker. These are used to -// constrain the new compactions that will be picked. +// Information about in-progress compactions provided to the compaction picker. +// These are used to constrain the new compactions that will be picked. type compactionInfo struct { - inputs []compactionLevel - outputLevel int - smallest InternalKey - largest InternalKey + // versionEditApplied is true if this compaction's version edit has already + // been committed. The compaction may still be in-progress deleting newly + // obsolete files. + versionEditApplied bool + inputs []compactionLevel + outputLevel int + smallest InternalKey + largest InternalKey } func (info compactionInfo) String() string { @@ -77,18 +90,38 @@ func (info compactionInfo) String() string { return buf.String() } -type sortCompactionLevelsDecreasingScore []candidateLevelInfo +type sortCompactionLevelsByPriority []candidateLevelInfo -func (s sortCompactionLevelsDecreasingScore) Len() int { +func (s sortCompactionLevelsByPriority) Len() int { return len(s) } -func (s sortCompactionLevelsDecreasingScore) Less(i, j int) bool { - if s[i].score != s[j].score { - return s[i].score > s[j].score + +// A level should be picked for compaction if the compensatedScoreRatio is >= the +// compactionScoreThreshold. +const compactionScoreThreshold = 1 + +// Less should return true if s[i] must be placed earlier than s[j] in the final +// sorted list. The candidateLevelInfo for the level placed earlier is more likely +// to be picked for a compaction. +func (s sortCompactionLevelsByPriority) Less(i, j int) bool { + iShouldCompact := s[i].compensatedScoreRatio >= compactionScoreThreshold + jShouldCompact := s[j].compensatedScoreRatio >= compactionScoreThreshold + // Ordering is defined as decreasing on (shouldCompact, uncompensatedScoreRatio) + // where shouldCompact is 1 for true and 0 for false. + if iShouldCompact && !jShouldCompact { + return true + } + if !iShouldCompact && jShouldCompact { + return false + } + + if s[i].uncompensatedScoreRatio != s[j].uncompensatedScoreRatio { + return s[i].uncompensatedScoreRatio > s[j].uncompensatedScoreRatio } return s[i].level < s[j].level } -func (s sortCompactionLevelsDecreasingScore) Swap(i, j int) { + +func (s sortCompactionLevelsByPriority) Swap(i, j int) { s[i], s[j] = s[j], s[i] } @@ -138,49 +171,43 @@ func generateSublevelInfo(cmp base.Compare, levelFiles manifest.LevelSlice) []su return levelSlices } +// compactionPickerMetrics holds metrics related to the compaction picking process +type compactionPickerMetrics struct { + // scores contains the compensatedScoreRatio from the candidateLevelInfo. + scores []float64 + singleLevelOverlappingRatio float64 + multiLevelOverlappingRatio float64 +} + // pickedCompaction contains information about a compaction that has already // been chosen, and is being constructed. Compaction construction info lives in // this struct, and is copied over into the compaction struct when that's // created. type pickedCompaction struct { cmp Compare - - // score of the chosen compaction. Taken from candidateLevelInfo. + // score of the chosen compaction. This is the same as the + // compensatedScoreRatio in the candidateLevelInfo. score float64 - // kind indicates the kind of compaction. kind compactionKind - // startLevel is the level that is being compacted. Inputs from startLevel // and outputLevel will be merged to produce a set of outputLevel files. startLevel *compactionLevel - // outputLevel is the level that files are being produced in. outputLevel is // equal to startLevel+1 except when: // - if startLevel is 0, the output level equals compactionPicker.baseLevel(). // - in multilevel compaction, the output level is the lowest level involved in // the compaction outputLevel *compactionLevel - // extraLevels contain additional levels in between the input and output // levels that get compacted in multi level compactions extraLevels []*compactionLevel - - // adjustedOutputLevel is the output level used for the purpose of - // determining the target output file size, overlap bytes, and expanded - // bytes, taking into account the base level. - adjustedOutputLevel int - - inputs []compactionLevel - + inputs []compactionLevel + // LBase at the time of compaction picking. + baseLevel int // L0-specific compaction info. Set to a non-nil value for all compactions // where startLevel == 0 that were generated by L0Sublevels. lcf *manifest.L0CompactionFiles - - // L0SublevelInfo is used for compactions out of L0. It is nil for all - // other compactions. - l0SublevelInfo []sublevelInfo - // maxOutputFileSize is the maximum size of an individual table created // during compaction. maxOutputFileSize uint64 @@ -191,12 +218,11 @@ type pickedCompaction struct { // overlap in its output level with. If the overlap is greater than // maxReadCompaction bytes, then we don't proceed with the compaction. maxReadCompactionBytes uint64 - // The boundaries of the input data. - smallest InternalKey - largest InternalKey - - version *version + smallest InternalKey + largest InternalKey + version *version + pickerMetrics compactionPickerMetrics } func defaultOutputLevel(startLevel, baseLevel int) int { @@ -218,28 +244,35 @@ func newPickedCompaction( startLevel, baseLevel)) } - adjustedOutputLevel := outputLevel - if adjustedOutputLevel > 0 { - // Output level is in the range [baseLevel,numLevels]. For the purpose of - // determining the target output file size, overlap bytes, and expanded - // bytes, we want to adjust the range to [1,numLevels]. - adjustedOutputLevel = 1 + outputLevel - baseLevel - } - + adjustedLevel := adjustedOutputLevel(outputLevel, baseLevel) pc := &pickedCompaction{ cmp: opts.Comparer.Compare, version: cur, + baseLevel: baseLevel, inputs: []compactionLevel{{level: startLevel}, {level: outputLevel}}, - adjustedOutputLevel: adjustedOutputLevel, - maxOutputFileSize: uint64(opts.Level(adjustedOutputLevel).TargetFileSize), - maxOverlapBytes: maxGrandparentOverlapBytes(opts, adjustedOutputLevel), - maxReadCompactionBytes: maxReadCompactionBytes(opts, adjustedOutputLevel), + maxOutputFileSize: uint64(opts.Level(adjustedLevel).TargetFileSize), + maxOverlapBytes: maxGrandparentOverlapBytes(opts, adjustedLevel), + maxReadCompactionBytes: maxReadCompactionBytes(opts, adjustedLevel), } pc.startLevel = &pc.inputs[0] pc.outputLevel = &pc.inputs[1] return pc } +// adjustedOutputLevel is the output level used for the purpose of +// determining the target output file size, overlap bytes, and expanded +// bytes, taking into account the base level. +func adjustedOutputLevel(outputLevel int, baseLevel int) int { + adjustedOutputLevel := outputLevel + if adjustedOutputLevel > 0 { + // Output level is in the range [baseLevel, numLevels]. For the purpose of + // determining the target output file size, overlap bytes, and expanded + // bytes, we want to adjust the range to [1,numLevels]. + adjustedOutputLevel = 1 + outputLevel - baseLevel + } + return adjustedOutputLevel +} + func newPickedCompactionFromL0( lcf *manifest.L0CompactionFiles, opts *Options, vers *version, baseLevel int, isBase bool, ) *pickedCompaction { @@ -272,7 +305,7 @@ func (pc *pickedCompaction) String() string { var builder strings.Builder builder.WriteString(fmt.Sprintf(`Score=%f, `, pc.score)) builder.WriteString(fmt.Sprintf(`Kind=%s, `, pc.kind)) - builder.WriteString(fmt.Sprintf(`AdjustedOutputLevel=%d, `, pc.adjustedOutputLevel)) + builder.WriteString(fmt.Sprintf(`AdjustedOutputLevel=%d, `, adjustedOutputLevel(pc.outputLevel.level, pc.baseLevel))) builder.WriteString(fmt.Sprintf(`maxOutputFileSize=%d, `, pc.maxOutputFileSize)) builder.WriteString(fmt.Sprintf(`maxReadCompactionBytes=%d, `, pc.maxReadCompactionBytes)) builder.WriteString(fmt.Sprintf(`smallest=%s, `, pc.smallest)) @@ -282,7 +315,7 @@ func (pc *pickedCompaction) String() string { builder.WriteString(fmt.Sprintf(`startlevel=%s, `, pc.startLevel)) builder.WriteString(fmt.Sprintf(`outputLevel=%s, `, pc.outputLevel)) builder.WriteString(fmt.Sprintf(`extraLevels=%s, `, pc.extraLevels)) - builder.WriteString(fmt.Sprintf(`l0SublevelInfo=%s, `, pc.l0SublevelInfo)) + builder.WriteString(fmt.Sprintf(`l0SublevelInfo=%s, `, pc.startLevel.l0SublevelInfo)) builder.WriteString(fmt.Sprintf(`lcf=%s`, pc.lcf)) return builder.String() } @@ -296,13 +329,16 @@ func (pc *pickedCompaction) clone() *pickedCompaction { cmp: pc.cmp, score: pc.score, kind: pc.kind, - adjustedOutputLevel: pc.adjustedOutputLevel, + baseLevel: pc.baseLevel, maxOutputFileSize: pc.maxOutputFileSize, maxOverlapBytes: pc.maxOverlapBytes, maxReadCompactionBytes: pc.maxReadCompactionBytes, smallest: pc.smallest.Clone(), largest: pc.largest.Clone(), + // TODO(msbutler): properly clone picker metrics + pickerMetrics: pc.pickerMetrics, + // Both copies see the same manifest, therefore, it's ok for them to se // share the same pc. version. version: pc.version, @@ -321,9 +357,11 @@ func (pc *pickedCompaction) clone() *pickedCompaction { } } - newPC.l0SublevelInfo = make([]sublevelInfo, len(pc.l0SublevelInfo)) - for i := range pc.l0SublevelInfo { - newPC.l0SublevelInfo[i] = pc.l0SublevelInfo[i].Clone() + if len(pc.startLevel.l0SublevelInfo) > 0 { + newPC.startLevel.l0SublevelInfo = make([]sublevelInfo, len(pc.startLevel.l0SublevelInfo)) + for i := range pc.startLevel.l0SublevelInfo { + newPC.startLevel.l0SublevelInfo[i] = pc.startLevel.l0SublevelInfo[i].Clone() + } } if pc.lcf != nil { newPC.lcf = pc.lcf.Clone() @@ -369,7 +407,7 @@ func (pc *pickedCompaction) setupInputs( // growing a compaction results in a larger size, the original compaction // is used instead. maxExpandedBytes := expandedCompactionByteSizeLimit( - opts, pc.adjustedOutputLevel, diskAvailBytes, + opts, adjustedOutputLevel(pc.outputLevel.level, pc.baseLevel), diskAvailBytes, ) // Expand the initial inputs to a clean cut. @@ -464,7 +502,7 @@ func (pc *pickedCompaction) setupInputs( if pc.startLevel.level == 0 { // We don't change the input files for the compaction beyond this point. - pc.l0SublevelInfo = generateSublevelInfo(pc.cmp, pc.startLevel.files) + pc.startLevel.l0SublevelInfo = generateSublevelInfo(pc.cmp, pc.startLevel.files) } return true @@ -527,8 +565,6 @@ func (pc *pickedCompaction) setupMultiLevelCandidate(opts *Options, diskAvailByt pc.startLevel = &pc.inputs[0] pc.extraLevels = []*compactionLevel{&pc.inputs[1]} pc.outputLevel = &pc.inputs[2] - - pc.adjustedOutputLevel++ return pc.setupInputs(opts, diskAvailBytes, pc.extraLevels[len(pc.extraLevels)-1]) } @@ -639,17 +675,11 @@ func expandToAtomicUnit( } func newCompactionPicker( - v *version, - opts *Options, - inProgressCompactions []compactionInfo, - levelSizes [numLevels]int64, - diskAvailBytes func() uint64, + v *version, opts *Options, inProgressCompactions []compactionInfo, ) compactionPicker { p := &compactionPickerByScore{ - opts: opts, - vers: v, - levelSizes: levelSizes, - diskAvailBytes: diskAvailBytes, + opts: opts, + vers: v, } p.initLevelMaxBytes(inProgressCompactions) return p @@ -658,16 +688,23 @@ func newCompactionPicker( // Information about a candidate compaction level that has been identified by // the compaction picker. type candidateLevelInfo struct { - // The score of the level to be compacted, with compensated file sizes and - // adjustments. - score float64 - // The original score of the level to be compacted, before adjusting - // according to other levels' sizes. - origScore float64 - // The raw score of the level to be compacted, calculated using - // uncompensated file sizes and without any adjustments. - rawScore float64 - level int + // The compensatedScore of the level after adjusting according to the other + // levels' sizes. For L0, the compensatedScoreRatio is equivalent to the + // uncompensatedScoreRatio as we don't account for level size compensation in + // L0. + compensatedScoreRatio float64 + // The score of the level after accounting for level size compensation before + // adjusting according to other levels' sizes. For L0, the compensatedScore + // is equivalent to the uncompensatedScore as we don't account for level + // size compensation in L0. + compensatedScore float64 + // The score of the level to be compacted, calculated using uncompensated file + // sizes and without any adjustments. + uncompensatedScore float64 + // uncompensatedScoreRatio is the uncompensatedScore adjusted according to + // the other levels' sizes. + uncompensatedScoreRatio float64 + level int // The level to compact to. outputLevel int // The file in level that will be compacted. Additional files may be @@ -676,6 +713,10 @@ type candidateLevelInfo struct { file manifest.LevelFile } +func (c *candidateLevelInfo) shouldCompact() bool { + return c.compensatedScoreRatio >= compactionScoreThreshold +} + func fileCompensation(f *fileMetadata) uint64 { return uint64(f.Stats.PointDeletionsBytesEstimate) + f.Stats.RangeDeletionsBytesEstimate } @@ -712,7 +753,7 @@ func (a compensatedSizeAnnotator) Accumulate( ) (v interface{}, cacheOK bool) { vptr := dst.(*uint64) *vptr = *vptr + compensatedSize(f) - return vptr, f.StatsValidLocked() + return vptr, f.StatsValid() } func (a compensatedSizeAnnotator) Merge(src interface{}, dst interface{}) interface{} { @@ -740,42 +781,20 @@ func totalCompensatedSize(iter manifest.LevelIterator) uint64 { type compactionPickerByScore struct { opts *Options vers *version - // The level to target for L0 compactions. Levels L1 to baseLevel must be // empty. baseLevel int - - // estimatedMaxWAmp is the estimated maximum write amp per byte that is - // added to L0. - estimatedMaxWAmp float64 - // levelMaxBytes holds the dynamically adjusted max bytes setting for each // level. levelMaxBytes [numLevels]int64 - - // levelSizes holds the current size of each level. - levelSizes [numLevels]int64 - - // diskAvailBytes returns a cached statistic on the number of bytes - // available on disk, as reported by the filesystem. It's used to be more - // restrictive in expanding compactions if available disk space is - // limited. - // - // The cached value is updated whenever a file is deleted and - // whenever a compaction or flush completes. Since file removal is - // the primary means of reclaiming space, there is a rough bound on - // the statistic's staleness when available bytes is growing. - // Compactions and flushes are longer, slower operations and provide - // a much looser bound when available bytes is decreasing. - diskAvailBytes func() uint64 } var _ compactionPicker = &compactionPickerByScore{} func (p *compactionPickerByScore) getScores(inProgress []compactionInfo) [numLevels]float64 { var scores [numLevels]float64 - for _, info := range p.calculateScores(inProgress) { - scores[info.level] = info.score + for _, info := range p.calculateLevelScores(inProgress) { + scores[info.level] = info.compensatedScoreRatio } return scores } @@ -787,10 +806,6 @@ func (p *compactionPickerByScore) getBaseLevel() int { return p.baseLevel } -func (p *compactionPickerByScore) getEstimatedMaxWAmp() float64 { - return p.estimatedMaxWAmp -} - // estimatedCompactionDebt estimates the number of bytes which need to be // compacted before the LSM tree becomes stable. func (p *compactionPickerByScore) estimatedCompactionDebt(l0ExtraSize uint64) uint64 { @@ -800,20 +815,22 @@ func (p *compactionPickerByScore) estimatedCompactionDebt(l0ExtraSize uint64) ui // We assume that all the bytes in L0 need to be compacted to Lbase. This is // unlike the RocksDB logic that figures out whether L0 needs compaction. - bytesAddedToNextLevel := l0ExtraSize + uint64(p.levelSizes[0]) - nextLevelSize := uint64(p.levelSizes[p.baseLevel]) + bytesAddedToNextLevel := l0ExtraSize + p.vers.Levels[0].Size() + lbaseSize := p.vers.Levels[p.baseLevel].Size() var compactionDebt uint64 - if bytesAddedToNextLevel > 0 && nextLevelSize > 0 { + if bytesAddedToNextLevel > 0 && lbaseSize > 0 { // We only incur compaction debt if both L0 and Lbase contain data. If L0 // is empty, no compaction is necessary. If Lbase is empty, a move-based // compaction from L0 would occur. - compactionDebt += bytesAddedToNextLevel + nextLevelSize + compactionDebt += bytesAddedToNextLevel + lbaseSize } + // loop invariant: At the beginning of the loop, bytesAddedToNextLevel is the + // bytes added to `level` in the loop. for level := p.baseLevel; level < numLevels-1; level++ { - levelSize := nextLevelSize + bytesAddedToNextLevel - nextLevelSize = uint64(p.levelSizes[level+1]) + levelSize := p.vers.Levels[level].Size() + bytesAddedToNextLevel + nextLevelSize := p.vers.Levels[level+1].Size() if levelSize > uint64(p.levelMaxBytes[level]) { bytesAddedToNextLevel = levelSize - uint64(p.levelMaxBytes[level]) if nextLevelSize > 0 { @@ -824,9 +841,11 @@ func (p *compactionPickerByScore) estimatedCompactionDebt(l0ExtraSize uint64) ui // The next level contributes levelRatio * bytesAddedToNextLevel. compactionDebt += uint64(float64(bytesAddedToNextLevel) * (levelRatio + 1)) } + } else { + // We're not moving any bytes to the next level. + bytesAddedToNextLevel = 0 } } - return compactionDebt } @@ -851,13 +870,13 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp // Determine the first non-empty level and the total DB size. firstNonEmptyLevel := -1 - var dbSize int64 + var dbSize uint64 for level := 1; level < numLevels; level++ { - if p.levelSizes[level] > 0 { + if p.vers.Levels[level].Size() > 0 { if firstNonEmptyLevel == -1 { firstNonEmptyLevel = level } - dbSize += p.levelSizes[level] + dbSize += p.vers.Levels[level].Size() } } for _, c := range inProgressCompactions { @@ -886,20 +905,20 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp return } - dbSize += p.levelSizes[0] - bottomLevelSize := dbSize - dbSize/int64(p.opts.Experimental.LevelMultiplier) + dbSize += p.vers.Levels[0].Size() + bottomLevelSize := dbSize - dbSize/uint64(p.opts.Experimental.LevelMultiplier) curLevelSize := bottomLevelSize for level := numLevels - 2; level >= firstNonEmptyLevel; level-- { - curLevelSize = int64(float64(curLevelSize) / float64(p.opts.Experimental.LevelMultiplier)) + curLevelSize = uint64(float64(curLevelSize) / float64(p.opts.Experimental.LevelMultiplier)) } // Compute base level (where L0 data is compacted to). - baseBytesMax := p.opts.LBaseMaxBytes + baseBytesMax := uint64(p.opts.LBaseMaxBytes) p.baseLevel = firstNonEmptyLevel for p.baseLevel > 1 && curLevelSize > baseBytesMax { p.baseLevel-- - curLevelSize = int64(float64(curLevelSize) / float64(p.opts.Experimental.LevelMultiplier)) + curLevelSize = uint64(float64(curLevelSize) / float64(p.opts.Experimental.LevelMultiplier)) } smoothedLevelMultiplier := 1.0 @@ -909,8 +928,6 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp 1.0/float64(numLevels-p.baseLevel-1)) } - p.estimatedMaxWAmp = float64(numLevels-p.baseLevel) * (smoothedLevelMultiplier + 1) - levelSize := float64(baseBytesMax) for level := p.baseLevel; level < numLevels; level++ { if level > p.baseLevel && levelSize > 0 { @@ -928,16 +945,16 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp } type levelSizeAdjust struct { - incomingActualBytes int64 - outgoingActualBytes int64 - outgoingCompensatedBytes int64 + incomingActualBytes uint64 + outgoingActualBytes uint64 + outgoingCompensatedBytes uint64 } -func (a levelSizeAdjust) compensated() int64 { +func (a levelSizeAdjust) compensated() uint64 { return a.incomingActualBytes - a.outgoingCompensatedBytes } -func (a levelSizeAdjust) actual() int64 { +func (a levelSizeAdjust) actual() uint64 { return a.incomingActualBytes - a.outgoingActualBytes } @@ -961,10 +978,16 @@ func calculateSizeAdjust(inProgressCompactions []compactionInfo) [numLevels]leve var sizeAdjust [numLevels]levelSizeAdjust for i := range inProgressCompactions { c := &inProgressCompactions[i] + // If this compaction's version edit has already been applied, there's + // no need to adjust: The LSM we'll examine will already reflect the + // new LSM state. + if c.versionEditApplied { + continue + } for _, input := range c.inputs { - actualSize := int64(input.files.SizeSum()) - compensatedSize := int64(totalCompensatedSize(input.files.Iter())) + actualSize := input.files.SizeSum() + compensatedSize := totalCompensatedSize(input.files.Iter()) if input.level != c.outputLevel { sizeAdjust[input.level].outgoingCompensatedBytes += compensatedSize @@ -982,7 +1005,7 @@ func levelCompensatedSize(lm manifest.LevelMetadata) uint64 { return *lm.Annotation(compensatedSizeAnnotator{}).(*uint64) } -func (p *compactionPickerByScore) calculateScores( +func (p *compactionPickerByScore) calculateLevelScores( inProgressCompactions []compactionInfo, ) [numLevels]candidateLevelInfo { var scores [numLevels]candidateLevelInfo @@ -990,72 +1013,89 @@ func (p *compactionPickerByScore) calculateScores( scores[i].level = i scores[i].outputLevel = i + 1 } - scores[0] = p.calculateL0Score(inProgressCompactions) - + l0UncompensatedScore := calculateL0UncompensatedScore(p.vers, p.opts, inProgressCompactions) + scores[0] = candidateLevelInfo{ + outputLevel: p.baseLevel, + uncompensatedScore: l0UncompensatedScore, + compensatedScore: l0UncompensatedScore, /* No level size compensation for L0 */ + } sizeAdjust := calculateSizeAdjust(inProgressCompactions) for level := 1; level < numLevels; level++ { - compensatedLevelSize := int64(levelCompensatedSize(p.vers.Levels[level])) + sizeAdjust[level].compensated() - scores[level].score = float64(compensatedLevelSize) / float64(p.levelMaxBytes[level]) - scores[level].origScore = scores[level].score - - // In addition to the compensated score, we calculate a separate score - // that uses actual file sizes, not compensated sizes. This is used - // during score smoothing down below to prevent excessive - // prioritization of reclaiming disk space. - scores[level].rawScore = float64(p.levelSizes[level]+sizeAdjust[level].actual()) / float64(p.levelMaxBytes[level]) + compensatedLevelSize := levelCompensatedSize(p.vers.Levels[level]) + sizeAdjust[level].compensated() + scores[level].compensatedScore = float64(compensatedLevelSize) / float64(p.levelMaxBytes[level]) + scores[level].uncompensatedScore = float64(p.vers.Levels[level].Size()+sizeAdjust[level].actual()) / float64(p.levelMaxBytes[level]) } - // Adjust each level's score by the score of the next level. If the next - // level has a high score, and is thus a priority for compaction, this - // reduces the priority for compacting the current level. If the next level - // has a low score (i.e. it is below its target size), this increases the - // priority for compacting the current level. + // Adjust each level's {compensated, uncompensated}Score by the uncompensatedScore + // of the next level to get a {compensated, uncompensated}ScoreRatio. If the + // next level has a high uncompensatedScore, and is thus a priority for compaction, + // this reduces the priority for compacting the current level. If the next level + // has a low uncompensatedScore (i.e. it is below its target size), this increases + // the priority for compacting the current level. // // The effect of this adjustment is to help prioritize compactions in lower - // levels. The following shows the new score and original score. In this - // scenario, L0 has 68 sublevels. L3 (a.k.a. Lbase) is significantly above - // its target size. The original score prioritizes compactions from those two - // levels, but doing so ends up causing a future problem: data piles up in - // the higher levels, starving L5->L6 compactions, and to a lesser degree - // starving L4->L5 compactions. + // levels. The following example shows the compensatedScoreRatio and the + // compensatedScore. In this scenario, L0 has 68 sublevels. L3 (a.k.a. Lbase) + // is significantly above its target size. The original score prioritizes + // compactions from those two levels, but doing so ends up causing a future + // problem: data piles up in the higher levels, starving L5->L6 compactions, + // and to a lesser degree starving L4->L5 compactions. // - // adjusted original - // score score size max-size - // L0 3.2 68.0 2.2 G - - // L3 3.2 21.1 1.3 G 64 M - // L4 3.4 6.7 3.1 G 467 M - // L5 3.4 2.0 6.6 G 3.3 G - // L6 0.6 0.6 14 G 24 G + // Note that in the example shown there is no level size compensation so the + // compensatedScore and the uncompensatedScore is the same for each level. + // + // compensatedScoreRatio compensatedScore uncompensatedScore size max-size + // L0 3.2 68.0 68.0 2.2 G - + // L3 3.2 21.1 21.1 1.3 G 64 M + // L4 3.4 6.7 6.7 3.1 G 467 M + // L5 3.4 2.0 2.0 6.6 G 3.3 G + // L6 0.6 0.6 0.6 14 G 24 G var prevLevel int for level := p.baseLevel; level < numLevels; level++ { - if scores[prevLevel].score >= 1 { - // Avoid absurdly large scores by placing a floor on the score that we'll - // adjust a level by. The value of 0.01 was chosen somewhat arbitrarily - const minScore = 0.01 - if scores[level].rawScore >= minScore { - scores[prevLevel].score /= scores[level].rawScore + // The compensated scores, and uncompensated scores will be turned into + // ratios as they're adjusted according to other levels' sizes. + scores[prevLevel].compensatedScoreRatio = scores[prevLevel].compensatedScore + scores[prevLevel].uncompensatedScoreRatio = scores[prevLevel].uncompensatedScore + + // Avoid absurdly large scores by placing a floor on the score that we'll + // adjust a level by. The value of 0.01 was chosen somewhat arbitrarily. + const minScore = 0.01 + if scores[prevLevel].compensatedScoreRatio >= compactionScoreThreshold { + if scores[level].uncompensatedScore >= minScore { + scores[prevLevel].compensatedScoreRatio /= scores[level].uncompensatedScore } else { - scores[prevLevel].score /= minScore + scores[prevLevel].compensatedScoreRatio /= minScore + } + } + if scores[prevLevel].uncompensatedScoreRatio >= compactionScoreThreshold { + if scores[level].uncompensatedScore >= minScore { + scores[prevLevel].uncompensatedScoreRatio /= scores[level].uncompensatedScore + } else { + scores[prevLevel].uncompensatedScoreRatio /= minScore } } prevLevel = level } + // Set the score ratios for the lowest level. + // INVARIANT: prevLevel == numLevels-1 + scores[prevLevel].compensatedScoreRatio = scores[prevLevel].compensatedScore + scores[prevLevel].uncompensatedScoreRatio = scores[prevLevel].uncompensatedScore - sort.Sort(sortCompactionLevelsDecreasingScore(scores[:])) + sort.Sort(sortCompactionLevelsByPriority(scores[:])) return scores } -func (p *compactionPickerByScore) calculateL0Score( - inProgressCompactions []compactionInfo, -) candidateLevelInfo { - var info candidateLevelInfo - info.outputLevel = p.baseLevel - - // If L0Sublevels are present, use the sublevel count to calculate the - // score. The base vs intra-L0 compaction determination happens in pickAuto, - // not here. - info.score = float64(2*p.vers.L0Sublevels.MaxDepthAfterOngoingCompactions()) / - float64(p.opts.L0CompactionThreshold) +// calculateL0UncompensatedScore calculates a float score representing the +// relative priority of compacting L0. Level L0 is special in that files within +// L0 may overlap one another, so a different set of heuristics that take into +// account read amplification apply. +func calculateL0UncompensatedScore( + vers *version, opts *Options, inProgressCompactions []compactionInfo, +) float64 { + // Use the sublevel count to calculate the score. The base vs intra-L0 + // compaction determination happens in pickAuto, not here. + score := float64(2*vers.L0Sublevels.MaxDepthAfterOngoingCompactions()) / + float64(opts.L0CompactionThreshold) // Also calculate a score based on the file count but use it only if it // produces a higher score than the sublevel-based one. This heuristic is @@ -1065,7 +1105,7 @@ func (p *compactionPickerByScore) calculateL0Score( // 1) we can produce a massive backlog to compact once files do overlap. // 2) constructing L0 sublevels has a runtime that grows superlinearly with // the number of files in L0 and must be done while holding D.mu. - noncompactingFiles := p.vers.Levels[0].Len() + noncompactingFiles := vers.Levels[0].Len() for _, c := range inProgressCompactions { for _, cl := range c.inputs { if cl.level == 0 { @@ -1073,15 +1113,20 @@ func (p *compactionPickerByScore) calculateL0Score( } } } - fileScore := float64(noncompactingFiles) / float64(p.opts.L0CompactionFileThreshold) - if info.score < fileScore { - info.score = fileScore + fileScore := float64(noncompactingFiles) / float64(opts.L0CompactionFileThreshold) + if score < fileScore { + score = fileScore } - return info + return score } -func (p *compactionPickerByScore) pickFile( - level, outputLevel int, earliestSnapshotSeqNum uint64, +// pickCompactionSeedFile picks a file from `level` in the `vers` to build a +// compaction around. Currently, this function implements a heuristic similar to +// RocksDB's kMinOverlappingRatio, seeking to minimize write amplification. This +// function is linear with respect to the number of files in `level` and +// `outputLevel`. +func pickCompactionSeedFile( + vers *version, opts *Options, level, outputLevel int, earliestSnapshotSeqNum uint64, ) (manifest.LevelFile, bool) { // Select the file within the level to compact. We want to minimize write // amplification, but also ensure that deletes are propagated to the @@ -1105,9 +1150,9 @@ func (p *compactionPickerByScore) pickFile( // pick a seed file whose resulting compaction bounds do not overlap with // an in-progress compaction. - cmp := p.opts.Comparer.Compare - startIter := p.vers.Levels[level].Iter() - outputIter := p.vers.Levels[outputLevel].Iter() + cmp := opts.Comparer.Compare + startIter := vers.Levels[level].Iter() + outputIter := vers.Levels[outputLevel].Iter() var file manifest.LevelFile smallestRatio := uint64(math.MaxUint64) @@ -1116,14 +1161,20 @@ func (p *compactionPickerByScore) pickFile( for f := startIter.First(); f != nil; f = startIter.Next() { var overlappingBytes uint64 + compacting := f.IsCompacting() + if compacting { + // Move on if this file is already being compacted. We'll likely + // still need to move past the overlapping output files regardless, + // but in cases where all start-level files are compacting we won't. + continue + } // Trim any output-level files smaller than f. - for outputFile != nil && base.InternalCompare(cmp, outputFile.Largest, f.Smallest) < 0 { + for outputFile != nil && sstableKeyCompare(cmp, outputFile.Largest, f.Smallest) < 0 { outputFile = outputIter.Next() } - compacting := f.IsCompacting() - for outputFile != nil && base.InternalCompare(cmp, outputFile.Smallest, f.Largest) < 0 { + for outputFile != nil && sstableKeyCompare(cmp, outputFile.Smallest, f.Largest) <= 0 && !compacting { overlappingBytes += outputFile.Size compacting = compacting || outputFile.IsCompacting() @@ -1142,7 +1193,32 @@ func (p *compactionPickerByScore) pickFile( // If the file in the next level extends beyond f's largest key, // break out and don't advance outputIter because f's successor // might also overlap. - if base.InternalCompare(cmp, outputFile.Largest, f.Largest) > 0 { + // + // Note, we stop as soon as we encounter an output-level file with a + // largest key beyond the input-level file's largest bound. We + // perform a simple user key comparison here using sstableKeyCompare + // which handles the potential for exclusive largest key bounds. + // There's some subtlety when the bounds are equal (eg, equal and + // inclusive, or equal and exclusive). Current Pebble doesn't split + // user keys across sstables within a level (and in format versions + // FormatSplitUserKeysMarkedCompacted and later we guarantee no + // split user keys exist within the entire LSM). In that case, we're + // assured that neither the input level nor the output level's next + // file shares the same user key, so compaction expansion will not + // include them in any compaction compacting `f`. + // + // NB: If we /did/ allow split user keys, or we're running on an + // old database with an earlier format major version where there are + // existing split user keys, this logic would be incorrect. Consider + // L1: [a#120,a#100] [a#80,a#60] + // L2: [a#55,a#45] [a#35,a#25] [a#15,a#5] + // While considering the first file in L1, [a#120,a#100], we'd skip + // past all of the files in L2. When considering the second file in + // L1, we'd improperly conclude that the second file overlaps + // nothing in the second level and is cheap to compact, when in + // reality we'd need to expand the compaction to include all 5 + // files. + if sstableKeyCompare(cmp, outputFile.Largest, f.Largest) > 0 { break } outputFile = outputIter.Next() @@ -1157,7 +1233,7 @@ func (p *compactionPickerByScore) pickFile( compSz := compensatedSize(f) scaledRatio := overlappingBytes * 1024 / compSz - if scaledRatio < smallestRatio && !f.IsCompacting() { + if scaledRatio < smallestRatio { smallestRatio = scaledRatio file = startIter.Take() } @@ -1185,15 +1261,15 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact // bytes have been compacted further down the LSM. if n := len(env.inProgressCompactions); n > 0 { l0ReadAmp := p.vers.L0Sublevels.MaxDepthAfterOngoingCompactions() - compactionDebt := int(p.estimatedCompactionDebt(0)) + compactionDebt := p.estimatedCompactionDebt(0) ccSignal1 := n * p.opts.Experimental.L0CompactionConcurrency - ccSignal2 := n * p.opts.Experimental.CompactionDebtConcurrency + ccSignal2 := uint64(n) * p.opts.Experimental.CompactionDebtConcurrency if l0ReadAmp < ccSignal1 && compactionDebt < ccSignal2 { return nil } } - scores := p.calculateScores(env.inProgressCompactions) + scores := p.calculateLevelScores(env.inProgressCompactions) // TODO(bananabrick): Either remove, or change this into an event sent to the // EventListener. @@ -1216,12 +1292,13 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact if pc.startLevel.level == info.level { marker = "*" } - fmt.Fprintf(&buf, " %sL%d: %5.1f %5.1f %5.1f %8s %8s", - marker, info.level, info.score, info.origScore, info.rawScore, - humanize.Int64(int64(totalCompensatedSize( + fmt.Fprintf(&buf, " %sL%d: %5.1f %5.1f %5.1f %5.1f %8s %8s", + marker, info.level, info.compensatedScoreRatio, info.compensatedScore, + info.uncompensatedScoreRatio, info.uncompensatedScore, + humanize.Bytes.Int64(int64(totalCompensatedSize( p.vers.Levels[info.level].Iter(), ))), - humanize.Int64(p.levelMaxBytes[info.level]), + humanize.Bytes.Int64(p.levelMaxBytes[info.level]), ) count := 0 @@ -1247,12 +1324,12 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact pc.startLevel.level, pc.outputLevel.level, buf.String()) } - // Check for a score-based compaction. "scores" has been sorted in order of - // decreasing score. For each level with a score >= 1, we attempt to find a - // compaction anchored at at that level. + // Check for a score-based compaction. candidateLevelInfos are first sorted + // by whether they should be compacted, so if we find a level which shouldn't + // be compacted, we can break early. for i := range scores { info := &scores[i] - if info.score < 1 { + if !info.shouldCompact() { break } if info.level == numLevels-1 { @@ -1260,11 +1337,12 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact } if info.level == 0 { - pc = pickL0(env, p.opts, p.vers, p.baseLevel, p.diskAvailBytes) + pc = pickL0(env, p.opts, p.vers, p.baseLevel) // Fail-safe to protect against compacting the same sstable // concurrently. if pc != nil && !inputRangeAlreadyCompacting(env, pc) { - pc.score = info.score + p.addScoresToPickedCompactionMetrics(pc, scores) + pc.score = info.compensatedScoreRatio // TODO(bananabrick): Create an EventListener for logCompaction. if false { logCompaction(pc) @@ -1276,15 +1354,16 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact // info.level > 0 var ok bool - info.file, ok = p.pickFile(info.level, info.outputLevel, env.earliestSnapshotSeqNum) + info.file, ok = pickCompactionSeedFile(p.vers, p.opts, info.level, info.outputLevel, env.earliestSnapshotSeqNum) if !ok { continue } - pc := pickAutoLPositive(env, p.opts, p.vers, *info, p.baseLevel, p.diskAvailBytes, p.levelMaxBytes) + pc := pickAutoLPositive(env, p.opts, p.vers, *info, p.baseLevel, p.levelMaxBytes) // Fail-safe to protect against compacting the same sstable concurrently. if pc != nil && !inputRangeAlreadyCompacting(env, pc) { - pc.score = info.score + p.addScoresToPickedCompactionMetrics(pc, scores) + pc.score = info.compensatedScoreRatio // TODO(bananabrick): Create an EventListener for logCompaction. if false { logCompaction(pc) @@ -1345,6 +1424,30 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact return nil } +func (p *compactionPickerByScore) addScoresToPickedCompactionMetrics( + pc *pickedCompaction, candInfo [numLevels]candidateLevelInfo, +) { + + // candInfo is sorted by score, not by compaction level. + infoByLevel := [numLevels]candidateLevelInfo{} + for i := range candInfo { + level := candInfo[i].level + infoByLevel[level] = candInfo[i] + } + // Gather the compaction scores for the levels participating in the compaction. + pc.pickerMetrics.scores = make([]float64, len(pc.inputs)) + inputIdx := 0 + for i := range infoByLevel { + if pc.inputs[inputIdx].level == infoByLevel[i].level { + pc.pickerMetrics.scores[inputIdx] = infoByLevel[i].compensatedScoreRatio + inputIdx++ + } + if inputIdx == len(pc.inputs) { + break + } + } +} + // elisionOnlyAnnotator implements the manifest.Annotator interface, // annotating B-Tree nodes with the *fileMetadata of a file meeting the // obsolete keys criteria for an elision-only compaction within the subtree. @@ -1364,7 +1467,7 @@ func (a elisionOnlyAnnotator) Accumulate(f *fileMetadata, dst interface{}) (inte if f.IsCompacting() { return dst, true } - if !f.StatsValidLocked() { + if !f.StatsValid() { return dst, false } // Bottommost files are large and not worthwhile to compact just @@ -1543,7 +1646,7 @@ func (p *compactionPickerByScore) pickRewriteCompaction(env compactionEnv) (pc * // Fail-safe to protect against compacting the same sstable concurrently. if !inputRangeAlreadyCompacting(env, pc) { if pc.startLevel.level == 0 { - pc.l0SublevelInfo = generateSublevelInfo(pc.cmp, pc.startLevel.files) + pc.startLevel.l0SublevelInfo = generateSublevelInfo(pc.cmp, pc.startLevel.files) } return pc } @@ -1560,7 +1663,6 @@ func pickAutoLPositive( vers *version, cInfo candidateLevelInfo, baseLevel int, - diskAvailBytes func() uint64, levelMaxBytes [7]int64, ) (pc *pickedCompaction) { if cInfo.level == 0 { @@ -1583,46 +1685,124 @@ func pickAutoLPositive( } } - if !pc.setupInputs(opts, diskAvailBytes(), pc.startLevel) { + if !pc.setupInputs(opts, env.diskAvailBytes, pc.startLevel) { return nil } - return pc.maybeAddLevel(opts, diskAvailBytes()) + return pc.maybeAddLevel(opts, env.diskAvailBytes) } // maybeAddLevel maybe adds a level to the picked compaction. func (pc *pickedCompaction) maybeAddLevel(opts *Options, diskAvailBytes uint64) *pickedCompaction { + pc.pickerMetrics.singleLevelOverlappingRatio = pc.overlappingRatio() if pc.outputLevel.level == numLevels-1 { // Don't add a level if the current output level is in L6 return pc } + if !opts.Experimental.MultiLevelCompactionHeuristic.allowL0() && pc.startLevel.level == 0 { + return pc + } if pc.compactionSize() > expandedCompactionByteSizeLimit( - opts, pc.adjustedOutputLevel, diskAvailBytes) { + opts, adjustedOutputLevel(pc.outputLevel.level, pc.baseLevel), diskAvailBytes) { // Don't add a level if the current compaction exceeds the compaction size limit return pc } - return opts.Experimental.MultiLevelCompactionHueristic.pick(pc, opts, diskAvailBytes) + return opts.Experimental.MultiLevelCompactionHeuristic.pick(pc, opts, diskAvailBytes) } // MultiLevelHeuristic evaluates whether to add files from the next level into the compaction. type MultiLevelHeuristic interface { - // evaluate returns the preferred compaction. + // Evaluate returns the preferred compaction. pick(pc *pickedCompaction, opts *Options, diskAvailBytes uint64) *pickedCompaction + + // Returns if the heuristic allows L0 to be involved in ML compaction + allowL0() bool } // NoMultiLevel will never add an additional level to the compaction. type NoMultiLevel struct{} +var _ MultiLevelHeuristic = (*NoMultiLevel)(nil) + func (nml NoMultiLevel) pick( pc *pickedCompaction, opts *Options, diskAvailBytes uint64, ) *pickedCompaction { return pc } +func (nml NoMultiLevel) allowL0() bool { + return false +} + +func (pc *pickedCompaction) predictedWriteAmp() float64 { + var bytesToCompact uint64 + var higherLevelBytes uint64 + for i := range pc.inputs { + levelSize := pc.inputs[i].files.SizeSum() + bytesToCompact += levelSize + if i != len(pc.inputs)-1 { + higherLevelBytes += levelSize + } + } + return float64(bytesToCompact) / float64(higherLevelBytes) +} + +func (pc *pickedCompaction) overlappingRatio() float64 { + var higherLevelBytes uint64 + var lowestLevelBytes uint64 + for i := range pc.inputs { + levelSize := pc.inputs[i].files.SizeSum() + if i == len(pc.inputs)-1 { + lowestLevelBytes += levelSize + continue + } + higherLevelBytes += levelSize + } + return float64(lowestLevelBytes) / float64(higherLevelBytes) +} + +// WriteAmpHeuristic defines a multi level compaction heuristic which will add +// an additional level to the picked compaction if it reduces predicted write +// amp of the compaction + the addPropensity constant. +type WriteAmpHeuristic struct { + // addPropensity is a constant that affects the propensity to conduct multilevel + // compactions. If positive, a multilevel compaction may get picked even if + // the single level compaction has lower write amp, and vice versa. + AddPropensity float64 + + // AllowL0 if true, allow l0 to be involved in a ML compaction. + AllowL0 bool +} + +var _ MultiLevelHeuristic = (*WriteAmpHeuristic)(nil) + +// TODO(msbutler): microbenchmark the extent to which multilevel compaction +// picking slows down the compaction picking process. This should be as fast as +// possible since Compaction-picking holds d.mu, which prevents WAL rotations, +// in-progress flushes and compactions from completing, etc. Consider ways to +// deduplicate work, given that setupInputs has already been called. +func (wa WriteAmpHeuristic) pick( + pcOrig *pickedCompaction, opts *Options, diskAvailBytes uint64, +) *pickedCompaction { + pcMulti := pcOrig.clone() + if !pcMulti.setupMultiLevelCandidate(opts, diskAvailBytes) { + return pcOrig + } + picked := pcOrig + if pcMulti.predictedWriteAmp() <= pcOrig.predictedWriteAmp()+wa.AddPropensity { + picked = pcMulti + } + // Regardless of what compaction was picked, log the multilevelOverlapping ratio. + picked.pickerMetrics.multiLevelOverlappingRatio = pcMulti.overlappingRatio() + return picked +} + +func (wa WriteAmpHeuristic) allowL0() bool { + return wa.AllowL0 +} + // Helper method to pick compactions originating from L0. Uses information about // sublevels to generate a compaction. -func pickL0( - env compactionEnv, opts *Options, vers *version, baseLevel int, diskAvailBytes func() uint64, -) (pc *pickedCompaction) { +func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (pc *pickedCompaction) { // It is important to pass information about Lbase files to L0Sublevels // so it can pick a compaction that does not conflict with an Lbase => Lbase+1 // compaction. Without this, we observed reduced concurrency of L0=>Lbase @@ -1637,11 +1817,11 @@ func pickL0( } if lcf != nil { pc = newPickedCompactionFromL0(lcf, opts, vers, baseLevel, true) - pc.setupInputs(opts, diskAvailBytes(), pc.startLevel) + pc.setupInputs(opts, env.diskAvailBytes, pc.startLevel) if pc.startLevel.files.Empty() { opts.Logger.Fatalf("empty compaction chosen") } - return pc + return pc.maybeAddLevel(opts, env.diskAvailBytes) } // Couldn't choose a base compaction. Try choosing an intra-L0 @@ -1655,7 +1835,7 @@ func pickL0( } if lcf != nil { pc = newPickedCompactionFromL0(lcf, opts, vers, 0, false) - if !pc.setupInputs(opts, diskAvailBytes(), pc.startLevel) { + if !pc.setupInputs(opts, env.diskAvailBytes, pc.startLevel) { return nil } if pc.startLevel.files.Empty() { @@ -1674,17 +1854,13 @@ func pickL0( return pc } -func (p *compactionPickerByScore) pickManual( - env compactionEnv, manual *manualCompaction, +func pickManualCompaction( + vers *version, opts *Options, env compactionEnv, baseLevel int, manual *manualCompaction, ) (pc *pickedCompaction, retryLater bool) { - if p == nil { - return nil, false - } - outputLevel := manual.level + 1 if manual.level == 0 { - outputLevel = p.baseLevel - } else if manual.level < p.baseLevel { + outputLevel = baseLevel + } else if manual.level < baseLevel { // The start level for a compaction must be >= Lbase. A manual // compaction could have been created adhering to that condition, and // then an automatic compaction came in and compacted all of the @@ -1698,11 +1874,22 @@ func (p *compactionPickerByScore) pickManual( // it, the compaction is dropped due to pc.setupInputs returning false since // the input/output range is already being compacted, and the manual // compaction ends with a non-compacted LSM. - if conflictsWithInProgress(manual, outputLevel, env.inProgressCompactions, p.opts.Comparer.Compare) { + if conflictsWithInProgress(manual, outputLevel, env.inProgressCompactions, opts.Comparer.Compare) { return nil, true } - pc = pickManualHelper(p.opts, manual, p.vers, p.baseLevel, p.diskAvailBytes, p.levelMaxBytes) - if pc == nil { + pc = newPickedCompaction(opts, vers, manual.level, defaultOutputLevel(manual.level, baseLevel), baseLevel) + manual.outputLevel = pc.outputLevel.level + pc.startLevel.files = vers.Overlaps(manual.level, opts.Comparer.Compare, manual.start, manual.end, false) + if pc.startLevel.files.Empty() { + // Nothing to do + return nil, false + } + if !pc.setupInputs(opts, env.diskAvailBytes, pc.startLevel) { + // setupInputs returned false indicating there's a conflicting + // concurrent compaction. + return nil, true + } + if pc = pc.maybeAddLevel(opts, env.diskAvailBytes); pc == nil { return nil, false } if pc.outputLevel.level != outputLevel { @@ -1719,28 +1906,6 @@ func (p *compactionPickerByScore) pickManual( return pc, false } -func pickManualHelper( - opts *Options, - manual *manualCompaction, - vers *version, - baseLevel int, - diskAvailBytes func() uint64, - levelMaxBytes [7]int64, -) (pc *pickedCompaction) { - pc = newPickedCompaction(opts, vers, manual.level, defaultOutputLevel(manual.level, baseLevel), baseLevel) - manual.outputLevel = pc.outputLevel.level - cmp := opts.Comparer.Compare - pc.startLevel.files = vers.Overlaps(manual.level, cmp, manual.start, manual.end, false) - if pc.startLevel.files.Empty() { - // Nothing to do - return nil - } - if !pc.setupInputs(opts, diskAvailBytes(), pc.startLevel) { - return nil - } - return pc.maybeAddLevel(opts, diskAvailBytes()) -} - func (p *compactionPickerByScore) pickReadTriggeredCompaction( env compactionEnv, ) (pc *pickedCompaction) { @@ -1786,7 +1951,7 @@ func pickReadTriggeredCompactionHelper( pc = newPickedCompaction(p.opts, p.vers, rc.level, defaultOutputLevel(rc.level, p.baseLevel), p.baseLevel) pc.startLevel.files = overlapSlice - if !pc.setupInputs(p.opts, p.diskAvailBytes(), pc.startLevel) { + if !pc.setupInputs(p.opts, env.diskAvailBytes, pc.startLevel) { return nil } if inputRangeAlreadyCompacting(env, pc) { diff --git a/vendor/github.com/cockroachdb/pebble/db.go b/vendor/github.com/cockroachdb/pebble/db.go index 9ec2d8b..f0d0591 100644 --- a/vendor/github.com/cockroachdb/pebble/db.go +++ b/vendor/github.com/cockroachdb/pebble/db.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "os" + "strconv" "sync" "sync/atomic" "time" @@ -17,15 +18,19 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/arenaskl" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invalidating" "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/internal/manual" "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/remote" + "github.com/cockroachdb/pebble/rangekey" "github.com/cockroachdb/pebble/record" "github.com/cockroachdb/pebble/sstable" "github.com/cockroachdb/pebble/vfs" "github.com/cockroachdb/pebble/vfs/atomicfs" + "github.com/cockroachdb/tokenbucket" "github.com/prometheus/client_golang/prometheus" ) @@ -70,7 +75,7 @@ type Reader interface { // NewIter returns an iterator that is unpositioned (Iterator.Valid() will // return false). The iterator can be positioned via a call to SeekGE, // SeekLT, First or Last. - NewIter(o *IterOptions) *Iterator + NewIter(o *IterOptions) (*Iterator, error) // Close closes the Reader. It may or may not close any underlying io.Reader // or io.Writer, depending on how the DB was created. @@ -96,6 +101,22 @@ type Writer interface { // It is safe to modify the contents of the arguments after Delete returns. Delete(key []byte, o *WriteOptions) error + // DeleteSized behaves identically to Delete, but takes an additional + // argument indicating the size of the value being deleted. DeleteSized + // should be preferred when the caller has the expectation that there exists + // a single internal KV pair for the key (eg, the key has not been + // overwritten recently), and the caller knows the size of its value. + // + // DeleteSized will record the value size within the tombstone and use it to + // inform compaction-picking heuristics which strive to reduce space + // amplification in the LSM. This "calling your shot" mechanic allows the + // storage engine to more accurately estimate and reduce space + // amplification. + // + // It is safe to modify the contents of the arguments after DeleteSized + // returns. + DeleteSized(key []byte, valueSize uint32, _ *WriteOptions) error + // SingleDelete is similar to Delete in that it deletes the value for the given key. Like Delete, // it is a blind operation that will succeed even if the given key does not exist. // @@ -232,25 +253,22 @@ func (d defaultCPUWorkGranter) CPUWorkDone(_ CPUWorkHandle) {} // Comparer: myComparer, // }) type DB struct { - // WARNING: The following struct `atomic` contains fields which are accessed - // atomically. - // - // Go allocations are guaranteed to be 64-bit aligned which we take advantage - // of by placing the 64-bit fields which we access atomically at the beginning - // of the DB struct. For more information, see https://golang.org/pkg/sync/atomic/#pkg-note-BUG. - atomic struct { - // The count and size of referenced memtables. This includes memtables - // present in DB.mu.mem.queue, as well as memtables that have been flushed - // but are still referenced by an inuse readState. - memTableCount int64 - memTableReserved int64 // number of bytes reserved in the cache for memtables + // The count and size of referenced memtables. This includes memtables + // present in DB.mu.mem.queue, as well as memtables that have been flushed + // but are still referenced by an inuse readState, as well as up to one + // memTable waiting to be reused and stored in d.memTableRecycle. + memTableCount atomic.Int64 + memTableReserved atomic.Int64 // number of bytes reserved in the cache for memtables + // memTableRecycle holds a pointer to an obsolete memtable. The next + // memtable allocation will reuse this memtable if it has not already been + // recycled. + memTableRecycle atomic.Pointer[memTable] - // The size of the current log file (i.e. db.mu.log.queue[len(queue)-1]. - logSize uint64 + // The size of the current log file (i.e. db.mu.log.queue[len(queue)-1]. + logSize atomic.Uint64 - // The number of bytes available on disk. - diskAvailBytes uint64 - } + // The number of bytes available on disk. + diskAvailBytes atomic.Uint64 cacheID uint64 dirname string @@ -263,14 +281,14 @@ type DB struct { abbreviatedKey AbbreviatedKey // The threshold for determining when a batch is "large" and will skip being // inserted into a memtable. - largeBatchThreshold int + largeBatchThreshold uint64 // The current OPTIONS file number. - optionsFileNum FileNum + optionsFileNum base.DiskFileNum // The on-disk size of the current OPTIONS file. optionsFileSize uint64 // objProvider is used to access and manage SSTs. - objProvider *objstorage.Provider + objProvider objstorage.Provider fileLock *Lock dataDir vfs.File @@ -297,14 +315,7 @@ type DB struct { closed *atomic.Value closedCh chan struct{} - deletionPacer pacer - - // Async deletion jobs spawned by cleaners increment this WaitGroup, and - // call Done when completed. Once `d.mu.cleaning` is false, the db.Close() - // goroutine needs to call Wait on this WaitGroup to ensure all cleaning - // and deleting goroutines have finished running. As deletion goroutines - // could grab db.mu, it must *not* be held while deleters.Wait() is called. - deleters sync.WaitGroup + cleanupManager *cleanupManager // During an iterator close, we may asynchronously schedule read compactions. // We want to wait for those goroutines to finish, before closing the DB. @@ -382,9 +393,6 @@ type DB struct { } mem struct { - // Condition variable used to serialize memtable switching. See - // DB.makeRoomForWrite(). - cond sync.Cond // The current mutable memTable. mutable *memTable // Queue of flushables (the mutable memtable is at end). Elements are @@ -397,7 +405,7 @@ type DB struct { // is allocated up to Options.MemTableSize. This reduces the memory // footprint of memtables when lots of DB instances are used concurrently // in test environments. - nextSize int + nextSize uint64 } compact struct { @@ -416,6 +424,10 @@ type DB struct { // is at the start of the list. New entries are added to the end. manual []*manualCompaction // inProgress is the set of in-progress flushes and compactions. + // It's used in the calculation of some metrics and to initialize L0 + // sublevels' state. Some of the compactions contained within this + // map may have already committed an edit to the version but are + // lingering performing cleanup, like deleting obsolete files. inProgress map[*compaction]struct{} // rescheduleReadCompaction indicates to an iterator that a read compaction @@ -426,6 +438,9 @@ type DB struct { // compactions which we might have to perform. readCompactions readCompactionQueue + // The cumulative duration of all completed compactions since Open. + // Does not include flushes. + duration time.Duration // Flush throughput metric. flushWriteThroughput ThroughputMetric // The idle start time for the flush "loop", i.e., when the flushing @@ -433,20 +448,10 @@ type DB struct { noOngoingFlushStartTime time.Time } - cleaner struct { - // Condition variable used to signal the completion of a file cleaning - // operation or an increment to the value of disabled. File cleaning operations are - // serialized, and a caller trying to do a file cleaning operation may wait - // until the ongoing one is complete. - cond sync.Cond - // True when a file cleaning operation is in progress. False does not necessarily - // mean all cleaning jobs have completed; see the comment on d.deleters. - cleaning bool - // Non-zero when file cleaning is disabled. The disabled count acts as a - // reference count to prohibit file cleaning. See - // DB.{disable,Enable}FileDeletions(). - disabled int - } + // Non-zero when file cleaning is disabled. The disabled count acts as a + // reference count to prohibit file cleaning. See + // DB.{disable,Enable}FileDeletions(). + disableFileDeletions int snapshots struct { // The list of active snapshots. @@ -481,7 +486,8 @@ type DB struct { // job to validate one or more sstables. cond sync.Cond // pending is a slice of metadata for sstables waiting to be - // validated. + // validated. Only physical sstables should be added to the pending + // queue. pending []newFileEntry // validating is set to true when validation is running. validating bool @@ -490,6 +496,9 @@ type DB struct { // Normally equal to time.Now() but may be overridden in tests. timeNow func() time.Time + // the time at database Open; may be used to compute metrics like effective + // compaction concurrency + openedAt time.Time } var _ Reader = (*DB)(nil) @@ -497,11 +506,7 @@ var _ Writer = (*DB)(nil) // TestOnlyWaitForCleaning MUST only be used in tests. func (d *DB) TestOnlyWaitForCleaning() { - d.mu.Lock() - defer d.mu.Unlock() - for d.mu.cleaner.cleaning { - d.mu.cleaner.cond.Wait() - } + d.cleanupManager.Wait() } // Get gets the value for the given key. It returns ErrNotFound if the DB does @@ -543,7 +548,7 @@ func (d *DB) getInternal(key []byte, b *Batch, s *Snapshot) ([]byte, io.Closer, if s != nil { seqNum = s.seqNum } else { - seqNum = atomic.LoadUint64(&d.mu.versions.atomic.visibleSeqNum) + seqNum = d.mu.versions.visibleSeqNum.Load() } buf := getIterAllocPool.Get().(*getIterAlloc) @@ -551,8 +556,7 @@ func (d *DB) getInternal(key []byte, b *Batch, s *Snapshot) ([]byte, io.Closer, get := &buf.get *get = getIter{ logger: d.opts.Logger, - cmp: d.cmp, - equal: d.equal, + comparer: d.opts.Comparer, newIters: d.newIters, snapshot: seqNum, key: key, @@ -625,6 +629,30 @@ func (d *DB) Delete(key []byte, opts *WriteOptions) error { return nil } +// DeleteSized behaves identically to Delete, but takes an additional +// argument indicating the size of the value being deleted. DeleteSized +// should be preferred when the caller has the expectation that there exists +// a single internal KV pair for the key (eg, the key has not been +// overwritten recently), and the caller knows the size of its value. +// +// DeleteSized will record the value size within the tombstone and use it to +// inform compaction-picking heuristics which strive to reduce space +// amplification in the LSM. This "calling your shot" mechanic allows the +// storage engine to more accurately estimate and reduce space amplification. +// +// It is safe to modify the contents of the arguments after DeleteSized +// returns. +func (d *DB) DeleteSized(key []byte, valueSize uint32, opts *WriteOptions) error { + b := newBatch(d) + _ = b.DeleteSized(key, valueSize, opts) + if err := d.Apply(b, opts); err != nil { + return err + } + // Only release the batch on success. + b.release() + return nil +} + // SingleDelete adds an action to the batch that single deletes the entry for key. // See Writer.SingleDelete for more details on the semantics of SingleDelete. // @@ -776,7 +804,10 @@ func (d *DB) applyInternal(batch *Batch, opts *WriteOptions, noSyncWait bool) er if err := d.closed.Load(); err != nil { panic(err) } - if atomic.LoadUint32(&batch.applied) != 0 { + if batch.committing { + panic("pebble: batch already committing") + } + if batch.applied.Load() { panic("pebble: batch already applied") } if d.opts.ReadOnly { @@ -791,25 +822,34 @@ func (d *DB) applyInternal(batch *Batch, opts *WriteOptions, noSyncWait bool) er return errors.New("pebble: WAL disabled") } + if batch.minimumFormatMajorVersion != FormatMostCompatible { + if fmv := d.FormatMajorVersion(); fmv < batch.minimumFormatMajorVersion { + panic(fmt.Sprintf( + "pebble: batch requires at least format major version %d (current: %d)", + batch.minimumFormatMajorVersion, fmv, + )) + } + } + if batch.countRangeKeys > 0 { if d.split == nil { return errNoSplit } - if d.FormatMajorVersion() < FormatRangeKeys { - panic(fmt.Sprintf( - "pebble: range keys require at least format major version %d (current: %d)", - FormatRangeKeys, d.FormatMajorVersion(), - )) - } - // TODO(jackson): Assert that all range key operands are suffixless. } + batch.committing = true if batch.db == nil { - batch.refreshMemTableSize() + if err := batch.refreshMemTableSize(); err != nil { + return err + } } - if int(batch.memTableSize) >= d.largeBatchThreshold { - batch.flushable = newFlushableBatch(batch, d.opts.Comparer) + if batch.memTableSize >= d.largeBatchThreshold { + var err error + batch.flushable, err = newFlushableBatch(batch, d.opts.Comparer) + if err != nil { + return err + } } if err := d.commit.Commit(batch, sync, noSyncWait); err != nil { // There isn't much we can do on an error here. The commit pipeline will be @@ -882,7 +922,7 @@ func (d *DB) commitWrite(b *Batch, syncWG *sync.WaitGroup, syncErr *error) (*mem b.flushable.setSeqNum(b.SeqNum()) if !d.opts.DisableWAL { var err error - size, b.commitStats.WALQueueWaitDuration, err = d.mu.log.SyncRecord(repr, syncWG, syncErr) + size, err = d.mu.log.SyncRecord(repr, syncWG, syncErr) if err != nil { panic(err) } @@ -920,13 +960,13 @@ func (d *DB) commitWrite(b *Batch, syncWG *sync.WaitGroup, syncErr *error) (*mem } if b.flushable == nil { - size, b.commitStats.WALQueueWaitDuration, err = d.mu.log.SyncRecord(repr, syncWG, syncErr) + size, err = d.mu.log.SyncRecord(repr, syncWG, syncErr) if err != nil { panic(err) } } - atomic.StoreUint64(&d.atomic.logSize, uint64(size)) + d.logSize.Store(uint64(size)) return mem, err } @@ -947,12 +987,30 @@ var iterAllocPool = sync.Pool{ }, } +// snapshotIterOpts denotes snapshot-related iterator options when calling +// newIter. These are the possible cases for a snapshotIterOpts: +// - No snapshot: All fields are zero values. +// - Classic snapshot: Only `seqNum` is set. The latest readState will be used +// and the specified seqNum will be used as the snapshot seqNum. +// - EventuallyFileOnlySnapshot (EFOS) behaving as a classic snapshot. Only +// the `seqNum` is set. The latest readState will be used +// and the specified seqNum will be used as the snapshot seqNum. +// - EFOS in file-only state: Only `seqNum` and `vers` are set. All the +// relevant SSTs are referenced by the *version. +type snapshotIterOpts struct { + seqNum uint64 + vers *version +} + // newIter constructs a new iterator, merging in batch iterators as an extra // level. -func (d *DB) newIter(ctx context.Context, batch *Batch, s *Snapshot, o *IterOptions) *Iterator { +func (d *DB) newIter( + ctx context.Context, batch *Batch, sOpts snapshotIterOpts, o *IterOptions, +) *Iterator { if err := d.closed.Load(); err != nil { panic(err) } + seqNum := sOpts.seqNum if o.rangeKeys() { if d.FormatMajorVersion() < FormatRangeKeys { panic(fmt.Sprintf( @@ -964,7 +1022,7 @@ func (d *DB) newIter(ctx context.Context, batch *Batch, s *Snapshot, o *IterOpti if o != nil && o.RangeKeyMasking.Suffix != nil && o.KeyTypes != IterKeyTypePointsAndRanges { panic("pebble: range key masking requires IterKeyTypePointsAndRanges") } - if (batch != nil || s != nil) && (o != nil && o.OnlyReadGuaranteedDurable) { + if (batch != nil || seqNum != 0) && (o != nil && o.OnlyReadGuaranteedDurable) { // We could add support for OnlyReadGuaranteedDurable on snapshots if // there was a need: this would require checking that the sequence number // of the snapshot has been flushed, by comparing with @@ -974,15 +1032,19 @@ func (d *DB) newIter(ctx context.Context, batch *Batch, s *Snapshot, o *IterOpti // Grab and reference the current readState. This prevents the underlying // files in the associated version from being deleted if there is a current // compaction. The readState is unref'd by Iterator.Close(). - readState := d.loadReadState() + var readState *readState + if sOpts.vers == nil { + // NB: loadReadState() calls readState.ref(). + readState = d.loadReadState() + } else { + // s.vers != nil + sOpts.vers.Ref() + } // Determine the seqnum to read at after grabbing the read state (current and // memtables) above. - var seqNum uint64 - if s != nil { - seqNum = s.seqNum - } else { - seqNum = atomic.LoadUint64(&d.mu.versions.atomic.visibleSeqNum) + if seqNum == 0 { + seqNum = d.mu.versions.visibleSeqNum.Load() } // Bundle various structures under a single umbrella in order to allocate @@ -995,6 +1057,7 @@ func (d *DB) newIter(ctx context.Context, batch *Batch, s *Snapshot, o *IterOpti merge: d.merge, comparer: *d.opts.Comparer, readState: readState, + version: sOpts.vers, keyBuf: buf.keyBuf, prefixOrFullSeekKey: buf.prefixOrFullSeekKey, boundsBuf: buf.boundsBuf, @@ -1024,7 +1087,10 @@ func (d *DB) newIter(ctx context.Context, batch *Batch, s *Snapshot, o *IterOpti func finishInitializingIter(ctx context.Context, buf *iterAlloc) *Iterator { // Short-hand. dbi := &buf.dbi - memtables := dbi.readState.memtables + var memtables flushableList + if dbi.readState != nil { + memtables = dbi.readState.memtables + } if dbi.opts.OnlyReadGuaranteedDurable { memtables = nil } else { @@ -1086,6 +1152,7 @@ func finishInitializingIter(ctx context.Context, buf *iterAlloc) *Iterator { }, } dbi.iter = &dbi.lazyCombinedIter + dbi.iter = invalidating.MaybeWrapIfInvariants(dbi.iter) } else { dbi.lazyCombinedIter.combinedIterState = combinedIterState{ initialized: true, @@ -1106,7 +1173,11 @@ func finishInitializingIter(ctx context.Context, buf *iterAlloc) *Iterator { // dbi already had an initialized range key iterator, in case the point // iterator changed or the range key masking suffix changed. dbi.rangeKey.iiter.Init(&dbi.comparer, dbi.iter, dbi.rangeKey.rangeKeyIter, - &dbi.rangeKeyMasking, dbi.opts.LowerBound, dbi.opts.UpperBound) + keyspan.InterleavingIterOpts{ + Mask: &dbi.rangeKeyMasking, + LowerBound: dbi.opts.LowerBound, + UpperBound: dbi.opts.UpperBound, + }) dbi.iter = &dbi.rangeKey.iiter } } else { @@ -1128,55 +1199,89 @@ func finishInitializingIter(ctx context.Context, buf *iterAlloc) *Iterator { // method, while the range deletion deleting that key must be exposed using // visitRangeDel. Keys that would be masked by range key masking (if an // appropriate prefix were set) should be exposed, alongside the range key -// that would have masked it. +// that would have masked it. This method also collapses all point keys into +// one InternalKey; so only one internal key at most per user key is returned +// to visitPointKey. +// +// If visitSharedFile is not nil, ScanInternal iterates in skip-shared iteration +// mode. In this iteration mode, sstables in levels L5 and L6 are skipped, and +// their metadatas truncated to [lower, upper) and passed into visitSharedFile. +// ErrInvalidSkipSharedIteration is returned if visitSharedFile is not nil and an +// sstable in L5 or L6 is found that is not in shared storage according to +// provider.IsShared, or an sstable in those levels contains a newer key than the +// snapshot sequence number (only applicable for snapshot.ScanInternal). Examples +// of when this could happen could be if Pebble started writing sstables before a +// creator ID was set (as creator IDs are necessary to enable shared storage) +// resulting in some lower level SSTs being on non-shared storage. Skip-shared +// iteration is invalid in those cases. func (d *DB) ScanInternal( + ctx context.Context, lower, upper []byte, - visitPointKey func(key *InternalKey, value LazyValue) error, + visitPointKey func(key *InternalKey, value LazyValue, iterInfo IteratorLevel) error, visitRangeDel func(start, end []byte, seqNum uint64) error, - visitRangeKey func(start, end []byte, keys []keyspan.Key) error, + visitRangeKey func(start, end []byte, keys []rangekey.Key) error, + visitSharedFile func(sst *SharedSSTMeta) error, ) error { - iter := d.newInternalIter(nil /* snapshot */, &IterOptions{ - KeyTypes: IterKeyTypePointsAndRanges, - LowerBound: lower, - UpperBound: upper, - }) + scanInternalOpts := &scanInternalOptions{ + visitPointKey: visitPointKey, + visitRangeDel: visitRangeDel, + visitRangeKey: visitRangeKey, + visitSharedFile: visitSharedFile, + skipSharedLevels: visitSharedFile != nil, + IterOptions: IterOptions{ + KeyTypes: IterKeyTypePointsAndRanges, + LowerBound: lower, + UpperBound: upper, + }, + } + iter := d.newInternalIter(snapshotIterOpts{} /* snapshot */, scanInternalOpts) defer iter.close() - return scanInternalImpl(lower, iter, visitPointKey, visitRangeDel, visitRangeKey) + return scanInternalImpl(ctx, lower, upper, iter, scanInternalOpts) } -// NewInternalIter constructs and returns a new scanInternalIterator on this db. +// newInternalIter constructs and returns a new scanInternalIterator on this db. +// If o.skipSharedLevels is true, levels below sharedLevelsStart are *not* added +// to the internal iterator. // // TODO(bilal): This method has a lot of similarities with db.newIter as well as // finishInitializingIter. Both pairs of methods should be refactored to reduce // this duplication. -func (d *DB) newInternalIter(s *Snapshot, o *IterOptions) *scanInternalIterator { +func (d *DB) newInternalIter(sOpts snapshotIterOpts, o *scanInternalOptions) *scanInternalIterator { if err := d.closed.Load(); err != nil { panic(err) } // Grab and reference the current readState. This prevents the underlying // files in the associated version from being deleted if there is a current // compaction. The readState is unref'd by Iterator.Close(). - readState := d.loadReadState() + var readState *readState + if sOpts.vers == nil { + readState = d.loadReadState() + } + if sOpts.vers != nil { + sOpts.vers.Ref() + } // Determine the seqnum to read at after grabbing the read state (current and // memtables) above. - var seqNum uint64 - if s == nil { - seqNum = atomic.LoadUint64(&d.mu.versions.atomic.visibleSeqNum) - } else { - seqNum = s.seqNum + seqNum := sOpts.seqNum + if seqNum == 0 { + seqNum = d.mu.versions.visibleSeqNum.Load() } // Bundle various structures under a single umbrella in order to allocate // them together. buf := iterAllocPool.Get().(*iterAlloc) dbi := &scanInternalIterator{ + db: d, comparer: d.opts.Comparer, + merge: d.opts.Merger.Merge, readState: readState, + version: sOpts.vers, alloc: buf, newIters: d.newIters, newIterRangeKey: d.tableNewRangeKeyIter, seqNum: seqNum, + mergingIter: &buf.merging, } if o != nil { dbi.opts = *o @@ -1190,7 +1295,10 @@ func (d *DB) newInternalIter(s *Snapshot, o *IterOptions) *scanInternalIterator func finishInitializingInternalIter(buf *iterAlloc, i *scanInternalIterator) *scanInternalIterator { // Short-hand. - memtables := i.readState.memtables + var memtables flushableList + if i.readState != nil { + memtables = i.readState.memtables + } // We only need to read from memtables which contain sequence numbers older // than seqNum. Trim off newer memtables. for j := len(memtables) - 1; j >= 0; j-- { @@ -1205,19 +1313,18 @@ func finishInitializingInternalIter(buf *iterAlloc, i *scanInternalIterator) *sc // For internal iterators, we skip the lazy combined iteration optimization // entirely, and create the range key iterator stack directly. - if i.rangeKey == nil { - i.rangeKey = iterRangeKeyStateAllocPool.Get().(*iteratorRangeKeyState) - i.rangeKey.init(i.comparer.Compare, i.comparer.Split, &i.opts) - i.constructRangeKeyIter() - } else { - i.rangeKey.iterConfig.SetBounds(i.opts.LowerBound, i.opts.UpperBound) - } + i.rangeKey = iterRangeKeyStateAllocPool.Get().(*iteratorRangeKeyState) + i.rangeKey.init(i.comparer.Compare, i.comparer.Split, &i.opts.IterOptions) + i.constructRangeKeyIter() // Wrap the point iterator (currently i.iter) with an interleaving // iterator that interleaves range keys pulled from // i.rangeKey.rangeKeyIter. i.rangeKey.iiter.Init(i.comparer, i.iter, i.rangeKey.rangeKeyIter, - nil /* mask */, i.opts.LowerBound, i.opts.UpperBound) + keyspan.InterleavingIterOpts{ + LowerBound: i.opts.LowerBound, + UpperBound: i.opts.UpperBound, + }) i.iter = &i.rangeKey.iiter return i @@ -1249,7 +1356,10 @@ func (i *Iterator) constructPointIter( } numMergingLevels += len(memtables) - current := i.readState.current + current := i.version + if current == nil { + current = i.readState.current + } numMergingLevels += len(current.L0SublevelFiles) numLevelIters += len(current.L0SublevelFiles) for level := 1; level < len(current.Levels); level++ { @@ -1309,15 +1419,16 @@ func (i *Iterator) constructPointIter( levelsIndex := len(levels) mlevels = mlevels[:numMergingLevels] levels = levels[:numLevelIters] + i.opts.snapshotForHideObsoletePoints = buf.dbi.seqNum addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Level) { li := &levels[levelsIndex] - li.init( - ctx, i.opts, i.comparer.Compare, i.comparer.Split, i.newIters, files, level, internalOpts) + li.init(ctx, i.opts, &i.comparer, i.newIters, files, level, internalOpts) li.initRangeDel(&mlevels[mlevelsIndex].rangeDelIter) li.initBoundaryContext(&mlevels[mlevelsIndex].levelIterBoundaryContext) li.initCombinedIterState(&i.lazyCombinedIter.combinedIterState) - mlevels[mlevelsIndex].iter = li + mlevels[mlevelsIndex].levelIter = li + mlevels[mlevelsIndex].iter = invalidating.MaybeWrapIfInvariants(li) levelsIndex++ mlevelsIndex++ @@ -1343,7 +1454,7 @@ func (i *Iterator) constructPointIter( buf.merging.snapshot = i.seqNum buf.merging.batchSnapshot = i.batchSeqNum buf.merging.combinedIterState = &i.lazyCombinedIter.combinedIterState - i.pointIter = &buf.merging + i.pointIter = invalidating.MaybeWrapIfInvariants(&buf.merging) i.merging = &buf.merging } @@ -1353,6 +1464,12 @@ func (d *DB) NewBatch() *Batch { return newBatch(d) } +// NewBatchWithSize is mostly identical to NewBatch, but it will allocate the +// the specified memory space for the internal slice in advance. +func (d *DB) NewBatchWithSize(size int) *Batch { + return newBatchWithSize(d, size) +} + // NewIndexedBatch returns a new empty read-write batch. Any reads on the batch // will read from both the batch and the DB. If the batch is committed it will // be applied to the DB. An indexed batch is slower that a non-indexed batch @@ -1362,6 +1479,12 @@ func (d *DB) NewIndexedBatch() *Batch { return newIndexedBatch(d, d.opts.Comparer) } +// NewIndexedBatchWithSize is mostly identical to NewIndexedBatch, but it will +// allocate the the specified memory space for the internal slice in advance. +func (d *DB) NewIndexedBatchWithSize(size int) *Batch { + return newIndexedBatchWithSize(d, d.opts.Comparer, size) +} + // NewIter returns an iterator that is unpositioned (Iterator.Valid() will // return false). The iterator can be positioned via a call to SeekGE, SeekLT, // First or Last. The iterator provides a point-in-time view of the current DB @@ -1370,14 +1493,14 @@ func (d *DB) NewIndexedBatch() *Batch { // to maintain a long-lived point-in-time view of the DB state can lead to an // apparent memory and disk usage leak. Use snapshots (see NewSnapshot) for // point-in-time snapshots which avoids these problems. -func (d *DB) NewIter(o *IterOptions) *Iterator { +func (d *DB) NewIter(o *IterOptions) (*Iterator, error) { return d.NewIterWithContext(context.Background(), o) } // NewIterWithContext is like NewIter, and additionally accepts a context for // tracing. -func (d *DB) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterator { - return d.newIter(ctx, nil /* batch */, nil /* snapshot */, o) +func (d *DB) NewIterWithContext(ctx context.Context, o *IterOptions) (*Iterator, error) { + return d.newIter(ctx, nil /* batch */, snapshotIterOpts{}, o), nil } // NewSnapshot returns a point-in-time view of the current DB state. Iterators @@ -1396,13 +1519,36 @@ func (d *DB) NewSnapshot() *Snapshot { d.mu.Lock() s := &Snapshot{ db: d, - seqNum: atomic.LoadUint64(&d.mu.versions.atomic.visibleSeqNum), + seqNum: d.mu.versions.visibleSeqNum.Load(), } d.mu.snapshots.pushBack(s) d.mu.Unlock() return s } +// NewEventuallyFileOnlySnapshot returns a point-in-time view of the current DB +// state, similar to NewSnapshot, but with consistency constrained to the +// provided set of key ranges. See the comment at EventuallyFileOnlySnapshot for +// its semantics. +func (d *DB) NewEventuallyFileOnlySnapshot(keyRanges []KeyRange) *EventuallyFileOnlySnapshot { + if err := d.closed.Load(); err != nil { + panic(err) + } + + internalKeyRanges := make([]internalKeyRange, len(keyRanges)) + for i := range keyRanges { + if i > 0 && d.cmp(keyRanges[i-1].End, keyRanges[i].Start) > 0 { + panic("pebble: key ranges for eventually-file-only-snapshot not in order") + } + internalKeyRanges[i] = internalKeyRange{ + smallest: base.MakeInternalKey(keyRanges[i].Start, InternalKeySeqNumMax, InternalKeyKindMax), + largest: base.MakeExclusiveSentinelKey(InternalKeyKindRangeDelete, keyRanges[i].End), + } + } + + return d.makeEventuallyFileOnlySnapshot(keyRanges, internalKeyRanges) +} + // Close closes the DB. // // It is not safe to close a DB until all outstanding iterators are closed @@ -1494,25 +1640,26 @@ func (d *DB) Close() error { // replay. mem.readerUnrefLocked(false) } - if reserved := atomic.LoadInt64(&d.atomic.memTableReserved); reserved != 0 { + // If there's an unused, recycled memtable, we need to release its memory. + if obsoleteMemTable := d.memTableRecycle.Swap(nil); obsoleteMemTable != nil { + d.freeMemTable(obsoleteMemTable) + } + if reserved := d.memTableReserved.Load(); reserved != 0 { err = firstError(err, errors.Errorf("leaked memtable reservation: %d", errors.Safe(reserved))) } - // No more cleaning can start. Wait for any async cleaning to complete. - for d.mu.cleaner.cleaning { - d.mu.cleaner.cond.Wait() - } - // There may still be obsolete tables if an existing async cleaning job - // prevented a new cleaning job when a readState was unrefed. If needed, - // synchronously delete obsolete files. + // Since we called d.readState.val.unrefLocked() above, we are expected to + // manually schedule deletion of obsolete files. if len(d.mu.versions.obsoleteTables) > 0 { - d.deleteObsoleteFiles(d.mu.nextJobID, true /* waitForOngoing */) + d.deleteObsoleteFiles(d.mu.nextJobID) } - // Wait for all the deletion goroutines spawned by cleaning jobs to finish. + d.mu.Unlock() - d.deleters.Wait() d.compactionSchedulers.Wait() + // Wait for all cleaning jobs to finish. + d.cleanupManager.Close() + // Sanity check metrics. if invariants.Enabled { m := d.Metrics() @@ -1572,6 +1719,10 @@ func (d *DB) Compact(start, end []byte, parallelize bool) error { } } + keyRanges := make([]internalKeyRange, len(meta)) + for i := range meta { + keyRanges[i] = internalKeyRange{smallest: m.Smallest, largest: m.Largest} + } // Determine if any memtable overlaps with the compaction range. We wait for // any such overlap to flush (initiating a flush if necessary). mem, err := func() (*flushableEntry, error) { @@ -1581,7 +1732,7 @@ func (d *DB) Compact(start, end []byte, parallelize bool) error { // overlaps. for i := len(d.mu.mem.queue) - 1; i >= 0; i-- { mem := d.mu.mem.queue[i] - if ingestMemtableOverlaps(d.cmp, mem, meta) { + if ingestMemtableOverlaps(d.cmp, mem, keyRanges) { var err error if mem.flushable == d.mu.mem.mutable { // We have to hold both commitPipeline.mu and DB.mu when calling @@ -1691,6 +1842,30 @@ func (d *DB) splitManualCompaction( return splitCompactions } +// DownloadSpan is a key range passed to the Download method. +type DownloadSpan struct { + StartKey []byte + // EndKey is exclusive. + EndKey []byte +} + +// Download ensures that the LSM does not use any external sstables for the +// given key ranges. It does so by performing appropriate compactions so that +// all external data becomes available locally. +// +// Note that calling this method does not imply that all other compactions stop; +// it simply informs Pebble of a list of spans for which external data should be +// downloaded with high priority. +// +// The method returns once no external sstasbles overlap the given spans, the +// context is canceled, or an error is hit. +// +// TODO(radu): consider passing a priority/impact knob to express how important +// the download is (versus live traffic performance, LSM health). +func (d *DB) Download(ctx context.Context, spans []DownloadSpan) error { + return errors.Errorf("not implemented") +} + // Flush the memtable to stable storage. func (d *DB) Flush() error { flushDone, err := d.AsyncFlush() @@ -1734,9 +1909,16 @@ func (d *DB) Metrics() *Metrics { vers := d.mu.versions.currentVersion() *metrics = d.mu.versions.metrics metrics.Compact.EstimatedDebt = d.mu.versions.picker.estimatedCompactionDebt(0) - metrics.Compact.InProgressBytes = atomic.LoadInt64(&d.mu.versions.atomic.atomicInProgressBytes) + metrics.Compact.InProgressBytes = d.mu.versions.atomicInProgressBytes.Load() metrics.Compact.NumInProgress = int64(d.mu.compact.compactingCount) metrics.Compact.MarkedFiles = vers.Stats.MarkedForCompaction + metrics.Compact.Duration = d.mu.compact.duration + for c := range d.mu.compact.inProgress { + if c.kind != compactionKindFlush { + metrics.Compact.Duration += d.timeNow().Sub(c.beganAt) + } + } + for _, m := range d.mu.mem.queue { metrics.MemTable.Size += m.totalBytes() } @@ -1747,11 +1929,11 @@ func (d *DB) Metrics() *Metrics { metrics.Snapshots.PinnedKeys = d.mu.snapshots.cumulativePinnedCount metrics.Snapshots.PinnedSize = d.mu.snapshots.cumulativePinnedSize metrics.MemTable.Count = int64(len(d.mu.mem.queue)) - metrics.MemTable.ZombieCount = atomic.LoadInt64(&d.atomic.memTableCount) - metrics.MemTable.Count - metrics.MemTable.ZombieSize = uint64(atomic.LoadInt64(&d.atomic.memTableReserved)) - metrics.MemTable.Size + metrics.MemTable.ZombieCount = d.memTableCount.Load() - metrics.MemTable.Count + metrics.MemTable.ZombieSize = uint64(d.memTableReserved.Load()) - metrics.MemTable.Size metrics.WAL.ObsoleteFiles = int64(recycledLogsCount) metrics.WAL.ObsoletePhysicalSize = recycledLogSize - metrics.WAL.Size = atomic.LoadUint64(&d.atomic.logSize) + metrics.WAL.Size = d.logSize.Load() // The current WAL size (d.atomic.logSize) is the current logical size, // which may be less than the WAL's physical size if it was recycled. // The file sizes in d.mu.log.queue are updated to the physical size @@ -1788,6 +1970,17 @@ func (d *DB) Metrics() *Metrics { d.mu.versions.logLock() metrics.private.manifestFileSize = uint64(d.mu.versions.manifest.Size()) + metrics.Table.BackingTableCount = uint64(len(d.mu.versions.backingState.fileBackingMap)) + metrics.Table.BackingTableSize = d.mu.versions.backingState.fileBackingSize + if invariants.Enabled { + var totalSize uint64 + for _, backing := range d.mu.versions.backingState.fileBackingMap { + totalSize += backing.Size + } + if totalSize != metrics.Table.BackingTableSize { + panic("pebble: invalid backing table size accounting") + } + } d.mu.versions.logUnlock() metrics.LogWriter.FsyncLatency = d.mu.log.metrics.fsyncLatency @@ -1807,6 +2000,11 @@ func (d *DB) Metrics() *Metrics { metrics.BlockCache = d.opts.Cache.Metrics() metrics.TableCache, metrics.Filter = d.tableCache.metrics() metrics.TableIters = int64(d.tableCache.iterCount()) + + metrics.SecondaryCacheMetrics = d.objProvider.Metrics() + + metrics.Uptime = d.timeNow().Sub(d.openedAt) + return metrics } @@ -1814,6 +2012,12 @@ func (d *DB) Metrics() *Metrics { type sstablesOptions struct { // set to true will return the sstable properties in TableInfo withProperties bool + + // if set, return sstables that overlap the key range (end-exclusive) + start []byte + end []byte + + withApproximateSpanBytes bool } // SSTablesOption set optional parameter used by `DB.SSTables`. @@ -1829,11 +2033,65 @@ func WithProperties() SSTablesOption { } } -// SSTableInfo export manifest.TableInfo with sstable.Properties +// WithKeyRangeFilter ensures returned sstables overlap start and end (end-exclusive) +// if start and end are both nil these properties have no effect. +func WithKeyRangeFilter(start, end []byte) SSTablesOption { + return func(opt *sstablesOptions) { + opt.end = end + opt.start = start + } +} + +// WithApproximateSpanBytes enables capturing the approximate number of bytes that +// overlap the provided key span for each sstable. +// NOTE: this option can only be used with WithKeyRangeFilter and WithProperties +// provided. +func WithApproximateSpanBytes() SSTablesOption { + return func(opt *sstablesOptions) { + opt.withApproximateSpanBytes = true + } +} + +// BackingType denotes the type of storage backing a given sstable. +type BackingType int + +const ( + // BackingTypeLocal denotes an sstable stored on local disk according to the + // objprovider. This file is completely owned by us. + BackingTypeLocal BackingType = iota + // BackingTypeShared denotes an sstable stored on shared storage, created + // by this Pebble instance and possibly shared by other Pebble instances. + // These types of files have lifecycle managed by Pebble. + BackingTypeShared + // BackingTypeSharedForeign denotes an sstable stored on shared storage, + // created by a Pebble instance other than this one. These types of files have + // lifecycle managed by Pebble. + BackingTypeSharedForeign + // BackingTypeExternal denotes an sstable stored on external storage, + // not owned by any Pebble instance and with no refcounting/cleanup methods + // or lifecycle management. An example of an external file is a file restored + // from a backup. + BackingTypeExternal +) + +// SSTableInfo export manifest.TableInfo with sstable.Properties alongside +// other file backing info. type SSTableInfo struct { manifest.TableInfo + // Virtual indicates whether the sstable is virtual. + Virtual bool + // BackingSSTNum is the file number associated with backing sstable which + // backs the sstable associated with this SSTableInfo. If Virtual is false, + // then BackingSSTNum == FileNum. + BackingSSTNum base.FileNum + // BackingType is the type of storage backing this sstable. + BackingType BackingType + // Locator is the remote.Locator backing this sstable, if the backing type is + // not BackingTypeLocal. + Locator remote.Locator - // Properties is the sstable properties of this table. + // Properties is the sstable properties of this table. If Virtual is true, + // then the Properties are associated with the backing sst. Properties *sstable.Properties } @@ -1847,6 +2105,13 @@ func (d *DB) SSTables(opts ...SSTablesOption) ([][]SSTableInfo, error) { fn(opt) } + if opt.withApproximateSpanBytes && !opt.withProperties { + return nil, errors.Errorf("Cannot use WithApproximateSpanBytes without WithProperties option.") + } + if opt.withApproximateSpanBytes && (opt.start == nil || opt.end == nil) { + return nil, errors.Errorf("Cannot use WithApproximateSpanBytes without WithKeyRangeFilter option.") + } + // Grab and reference the current readState. readState := d.loadReadState() defer readState.unref() @@ -1867,19 +2132,67 @@ func (d *DB) SSTables(opts ...SSTablesOption) ([][]SSTableInfo, error) { iter := srcLevels[i].Iter() j := 0 for m := iter.First(); m != nil; m = iter.Next() { + if opt.start != nil && opt.end != nil && !m.Overlaps(d.opts.Comparer.Compare, opt.start, opt.end, true /* exclusive end */) { + continue + } destTables[j] = SSTableInfo{TableInfo: m.TableInfo()} if opt.withProperties { - p, err := d.tableCache.getTableProperties(m) + p, err := d.tableCache.getTableProperties( + m, + ) if err != nil { return nil, err } destTables[j].Properties = p } + destTables[j].Virtual = m.Virtual + destTables[j].BackingSSTNum = m.FileBacking.DiskFileNum.FileNum() + objMeta, err := d.objProvider.Lookup(fileTypeTable, m.FileBacking.DiskFileNum) + if err != nil { + return nil, err + } + if objMeta.IsRemote() { + if objMeta.IsShared() { + if d.objProvider.IsSharedForeign(objMeta) { + destTables[j].BackingType = BackingTypeSharedForeign + } else { + destTables[j].BackingType = BackingTypeShared + } + } else { + destTables[j].BackingType = BackingTypeExternal + } + destTables[j].Locator = objMeta.Remote.Locator + } else { + destTables[j].BackingType = BackingTypeLocal + } + + if opt.withApproximateSpanBytes { + var spanBytes uint64 + if m.ContainedWithinSpan(d.opts.Comparer.Compare, opt.start, opt.end) { + spanBytes = m.Size + } else { + size, err := d.tableCache.estimateSize(m, opt.start, opt.end) + if err != nil { + return nil, err + } + spanBytes = size + } + propertiesCopy := *destTables[j].Properties + + // Deep copy user properties so approximate span bytes can be added. + propertiesCopy.UserProperties = make(map[string]string, len(destTables[j].Properties.UserProperties)+1) + for k, v := range destTables[j].Properties.UserProperties { + propertiesCopy.UserProperties[k] = v + } + propertiesCopy.UserProperties["approximate-span-bytes"] = strconv.FormatUint(spanBytes, 10) + destTables[j].Properties = &propertiesCopy + } j++ } destLevels[i] = destTables[:j] destTables = destTables[j:] } + return destLevels, nil } @@ -1892,14 +2205,25 @@ func (d *DB) SSTables(opts ...SSTablesOption) ([][]SSTableInfo, error) { // overlap due to abbreviated index keys, the full data block size is included in // the estimation. Note that unlike fully contained sstables, none of the // meta-block space is counted for partially overlapped files. +// - For virtual sstables, we use the overlap between start, end and the virtual +// sstable bounds to determine disk usage. // - There may also exist WAL entries for unflushed keys in this range. This // estimation currently excludes space used for the range in the WAL. func (d *DB) EstimateDiskUsage(start, end []byte) (uint64, error) { + bytes, _, _, err := d.EstimateDiskUsageByBackingType(start, end) + return bytes, err +} + +// EstimateDiskUsageByBackingType is like EstimateDiskUsage but additionally +// returns the subsets of that size in remote ane external files. +func (d *DB) EstimateDiskUsageByBackingType( + start, end []byte, +) (totalSize, remoteSize, externalSize uint64, _ error) { if err := d.closed.Load(); err != nil { panic(err) } if d.opts.Comparer.Compare(start, end) > 0 { - return 0, errors.New("invalid key-range specified (start > end)") + return 0, 0, 0, errors.New("invalid key-range specified (start > end)") } // Grab and reference the current readState. This prevents the underlying @@ -1908,7 +2232,6 @@ func (d *DB) EstimateDiskUsage(start, end []byte) (uint64, error) { readState := d.loadReadState() defer readState.unref() - var totalSize uint64 for level, files := range readState.current.Levels { iter := files.Iter() if level > 0 { @@ -1923,22 +2246,56 @@ func (d *DB) EstimateDiskUsage(start, end []byte) (uint64, error) { d.opts.Comparer.Compare(file.Largest.UserKey, end) <= 0 { // The range fully contains the file, so skip looking it up in // table cache/looking at its indexes, and add the full file size. + meta, err := d.objProvider.Lookup(fileTypeTable, file.FileBacking.DiskFileNum) + if err != nil { + return 0, 0, 0, err + } + if meta.IsRemote() { + remoteSize += file.Size + if meta.Remote.CleanupMethod == objstorage.SharedNoCleanup { + externalSize += file.Size + } + } totalSize += file.Size } else if d.opts.Comparer.Compare(file.Smallest.UserKey, end) <= 0 && d.opts.Comparer.Compare(start, file.Largest.UserKey) <= 0 { var size uint64 - err := d.tableCache.withReader(file, func(r *sstable.Reader) (err error) { - size, err = r.EstimateDiskUsage(start, end) - return err - }) + var err error + if file.Virtual { + err = d.tableCache.withVirtualReader( + file.VirtualMeta(), + func(r sstable.VirtualReader) (err error) { + size, err = r.EstimateDiskUsage(start, end) + return err + }, + ) + } else { + err = d.tableCache.withReader( + file.PhysicalMeta(), + func(r *sstable.Reader) (err error) { + size, err = r.EstimateDiskUsage(start, end) + return err + }, + ) + } if err != nil { - return 0, err + return 0, 0, 0, err + } + meta, err := d.objProvider.Lookup(fileTypeTable, file.FileBacking.DiskFileNum) + if err != nil { + return 0, 0, 0, err + } + if meta.IsRemote() { + remoteSize += size + if meta.Remote.CleanupMethod == objstorage.SharedNoCleanup { + externalSize += size + } } totalSize += size } } } - return totalSize, nil + return totalSize, remoteSize, externalSize, nil } func (d *DB) walPreallocateSize() int { @@ -1952,7 +2309,7 @@ func (d *DB) walPreallocateSize() int { // RocksDB. Could a smaller preallocation block size be used? size := d.opts.MemTableSize size = (size / 10) + size - return size + return int(size) } func (d *DB) newMemTable(logNum FileNum, logSeqNum uint64) (*memTable, *flushableEntry) { @@ -1964,40 +2321,79 @@ func (d *DB) newMemTable(logNum FileNum, logSeqNum uint64) (*memTable, *flushabl } } - atomic.AddInt64(&d.atomic.memTableCount, 1) - atomic.AddInt64(&d.atomic.memTableReserved, int64(size)) - releaseAccountingReservation := d.opts.Cache.Reserve(size) - - mem := newMemTable(memTableOptions{ + memtblOpts := memTableOptions{ Options: d.opts, - arenaBuf: manual.New(int(size)), logSeqNum: logSeqNum, - }) + } - // Note: this is a no-op if invariants are disabled or race is enabled. - invariants.SetFinalizer(mem, checkMemTable) + // Before attempting to allocate a new memtable, check if there's one + // available for recycling in memTableRecycle. Large contiguous allocations + // can be costly as fragmentation makes it more difficult to find a large + // contiguous free space. We've observed 64MB allocations taking 10ms+. + // + // To reduce these costly allocations, up to 1 obsolete memtable is stashed + // in `d.memTableRecycle` to allow a future memtable rotation to reuse + // existing memory. + var mem *memTable + mem = d.memTableRecycle.Swap(nil) + if mem != nil && uint64(len(mem.arenaBuf)) != size { + d.freeMemTable(mem) + mem = nil + } + if mem != nil { + // Carry through the existing buffer and memory reservation. + memtblOpts.arenaBuf = mem.arenaBuf + memtblOpts.releaseAccountingReservation = mem.releaseAccountingReservation + } else { + mem = new(memTable) + memtblOpts.arenaBuf = manual.New(int(size)) + memtblOpts.releaseAccountingReservation = d.opts.Cache.Reserve(int(size)) + d.memTableCount.Add(1) + d.memTableReserved.Add(int64(size)) + + // Note: this is a no-op if invariants are disabled or race is enabled. + invariants.SetFinalizer(mem, checkMemTable) + } + mem.init(memtblOpts) entry := d.newFlushableEntry(mem, logNum, logSeqNum) entry.releaseMemAccounting = func() { - manual.Free(mem.arenaBuf) - mem.arenaBuf = nil - atomic.AddInt64(&d.atomic.memTableCount, -1) - atomic.AddInt64(&d.atomic.memTableReserved, -int64(size)) - releaseAccountingReservation() + // If the user leaks iterators, we may be releasing the memtable after + // the DB is already closed. In this case, we want to just release the + // memory because DB.Close won't come along to free it for us. + if err := d.closed.Load(); err != nil { + d.freeMemTable(mem) + return + } + + // The next memtable allocation might be able to reuse this memtable. + // Stash it on d.memTableRecycle. + if unusedMem := d.memTableRecycle.Swap(mem); unusedMem != nil { + // There was already a memtable waiting to be recycled. We're now + // responsible for freeing it. + d.freeMemTable(unusedMem) + } } return mem, entry } +func (d *DB) freeMemTable(m *memTable) { + d.memTableCount.Add(-1) + d.memTableReserved.Add(-int64(len(m.arenaBuf))) + m.free() +} + func (d *DB) newFlushableEntry(f flushable, logNum FileNum, logSeqNum uint64) *flushableEntry { - return &flushableEntry{ + fe := &flushableEntry{ flushable: f, flushed: make(chan struct{}), logNum: logNum, logSeqNum: logSeqNum, - readerRefs: 1, deleteFn: d.mu.versions.addObsolete, deleteFnLocked: d.mu.versions.addObsoleteLocked, } + fe.readerRefs.Store(1) + return fe } // makeRoomForWrite ensures that the memtable has room to hold the contents of @@ -2037,7 +2433,7 @@ func (d *DB) makeRoomForWrite(b *Batch) error { for i := range d.mu.mem.queue { size += d.mu.mem.queue[i].totalBytes() } - if size >= uint64(d.opts.MemTableStopWritesThreshold)*uint64(d.opts.MemTableSize) { + if size >= uint64(d.opts.MemTableStopWritesThreshold)*d.opts.MemTableSize { // We have filled up the current memtable, but already queued memtables // are still flushing, so we wait. if !stalled { @@ -2091,7 +2487,7 @@ func (d *DB) makeRoomForWrite(b *Batch) error { // reduces memtable memory pressure when an application is frequently // manually flushing. if (b == nil) && uint64(immMem.availBytes()) > immMem.totalBytes()/2 { - d.mu.mem.nextSize = int(immMem.totalBytes()) + d.mu.mem.nextSize = immMem.totalBytes() } if b != nil && b.flushable != nil { @@ -2120,7 +2516,7 @@ func (d *DB) makeRoomForWrite(b *Batch) error { logSeqNum += uint64(b.Count()) } } else { - logSeqNum = atomic.LoadUint64(&d.mu.versions.atomic.logSeqNum) + logSeqNum = d.mu.versions.logSeqNum.Load() } d.rotateMemtable(newLogNum, logSeqNum, immMem) force = false @@ -2188,7 +2584,7 @@ func (d *DB) recycleWAL() (newLogNum FileNum, prevLogSize uint64) { } d.mu.Unlock() - newLogName := base.MakeFilepath(d.opts.FS, d.walDirname, fileTypeLog, newLogNum) + newLogName := base.MakeFilepath(d.opts.FS, d.walDirname, fileTypeLog, newLogNum.DiskFileNum()) // Try to use a recycled log file. Recycling log files is an important // performance optimization as it is faster to sync a file that has @@ -2245,19 +2641,18 @@ func (d *DB) recycleWAL() (newLogNum FileNum, prevLogSize uint64) { } if recycleOK { - err = firstError(err, d.logRecycler.pop(recycleLog.fileNum)) + err = firstError(err, d.logRecycler.pop(recycleLog.fileNum.FileNum())) } d.opts.EventListener.WALCreated(WALCreateInfo{ JobID: jobID, Path: newLogName, FileNum: newLogNum, - RecycledFileNum: recycleLog.fileNum, + RecycledFileNum: recycleLog.fileNum.FileNum(), Err: err, }) d.mu.Lock() - d.mu.mem.cond.Broadcast() d.mu.versions.metrics.WAL.Files++ @@ -2270,7 +2665,7 @@ func (d *DB) recycleWAL() (newLogNum FileNum, prevLogSize uint64) { panic(err) } - d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum, fileSize: newLogSize}) + d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum.DiskFileNum(), fileSize: newLogSize}) d.mu.log.LogWriter = record.NewLogWriter(newLogFile, newLogNum, record.LogWriterConfig{ WALFsyncLatency: d.mu.log.metrics.fsyncLatency, WALMinSyncInterval: d.opts.WALMinSyncInterval, @@ -2298,10 +2693,11 @@ func (d *DB) getInProgressCompactionInfoLocked(finishing *compaction) (rv []comp for c := range d.mu.compact.inProgress { if len(c.flushing) == 0 && (finishing == nil || c != finishing) { info := compactionInfo{ - inputs: c.inputs, - smallest: c.smallest, - largest: c.largest, - outputLevel: -1, + versionEditApplied: c.versionEditApplied, + inputs: c.inputs, + smallest: c.smallest, + largest: c.largest, + outputLevel: -1, } if c.outputLevel != nil { info.outputLevel = c.outputLevel.level @@ -2315,6 +2711,14 @@ func (d *DB) getInProgressCompactionInfoLocked(finishing *compaction) (rv []comp func inProgressL0Compactions(inProgress []compactionInfo) []manifest.L0Compaction { var compactions []manifest.L0Compaction for _, info := range inProgress { + // Skip in-progress compactions that have already committed; the L0 + // sublevels initialization code requires the set of in-progress + // compactions to be consistent with the current version. Compactions + // with versionEditApplied=true are already applied to the current + // version and but are performing cleanup without the database mutex. + if info.versionEditApplied { + continue + } l0 := false for _, cl := range info.inputs { l0 = l0 || cl.level == 0 @@ -2341,14 +2745,246 @@ func firstError(err0, err1 error) error { } // SetCreatorID sets the CreatorID which is needed in order to use shared objects. -// Shared object usage is disabled until this method is called the first time. +// Remote object usage is disabled until this method is called the first time. // Once set, the Creator ID is persisted and cannot change. // // Does nothing if SharedStorage was not set in the options when the DB was // opened or if the DB is in read-only mode. func (d *DB) SetCreatorID(creatorID uint64) error { - if d.opts.Experimental.SharedStorage == nil || d.opts.ReadOnly { + if d.opts.Experimental.RemoteStorage == nil || d.opts.ReadOnly { return nil } return d.objProvider.SetCreatorID(objstorage.CreatorID(creatorID)) } + +// KeyStatistics keeps track of the number of keys that have been pinned by a +// snapshot as well as counts of the different key kinds in the lsm. +// +// One way of using the accumulated stats, when we only have sets and dels, +// and say the counts are represented as del_count, set_count, +// del_latest_count, set_latest_count, snapshot_pinned_count. +// +// - del_latest_count + set_latest_count is the set of unique user keys +// (unique). +// +// - set_latest_count is the set of live unique user keys (live_unique). +// +// - Garbage is del_count + set_count - live_unique. +// +// - If everything were in the LSM, del_count+set_count-snapshot_pinned_count +// would also be the set of unique user keys (note that +// snapshot_pinned_count is counting something different -- see comment below). +// But snapshot_pinned_count only counts keys in the LSM so the excess here +// must be keys in memtables. +type KeyStatistics struct { + // TODO(sumeer): the SnapshotPinned* are incorrect in that these older + // versions can be in a different level. Either fix the accounting or + // rename these fields. + + // SnapshotPinnedKeys represents obsolete keys that cannot be elided during + // a compaction, because they are required by an open snapshot. + SnapshotPinnedKeys int + // SnapshotPinnedKeysBytes is the total number of bytes of all snapshot + // pinned keys. + SnapshotPinnedKeysBytes uint64 + // KindsCount is the count for each kind of key. It includes point keys, + // range deletes and range keys. + KindsCount [InternalKeyKindMax + 1]int + // LatestKindsCount is the count for each kind of key when it is the latest + // kind for a user key. It is only populated for point keys. + LatestKindsCount [InternalKeyKindMax + 1]int +} + +// LSMKeyStatistics is used by DB.ScanStatistics. +type LSMKeyStatistics struct { + Accumulated KeyStatistics + // Levels contains statistics only for point keys. Range deletions and range keys will + // appear in Accumulated but not Levels. + Levels [numLevels]KeyStatistics + // BytesRead represents the logical, pre-compression size of keys and values read + BytesRead uint64 +} + +// ScanStatisticsOptions is used by DB.ScanStatistics. +type ScanStatisticsOptions struct { + // LimitBytesPerSecond indicates the number of bytes that are able to be read + // per second using ScanInternal. + // A value of 0 indicates that there is no limit set. + LimitBytesPerSecond int64 +} + +// ScanStatistics returns the count of different key kinds within the lsm for a +// key span [lower, upper) as well as the number of snapshot keys. +func (d *DB) ScanStatistics( + ctx context.Context, lower, upper []byte, opts ScanStatisticsOptions, +) (LSMKeyStatistics, error) { + stats := LSMKeyStatistics{} + var prevKey InternalKey + var rateLimitFunc func(key *InternalKey, val LazyValue) error + tb := tokenbucket.TokenBucket{} + + if opts.LimitBytesPerSecond != 0 { + // Each "token" roughly corresponds to a byte that was read. + tb.Init(tokenbucket.TokensPerSecond(opts.LimitBytesPerSecond), tokenbucket.Tokens(1024)) + rateLimitFunc = func(key *InternalKey, val LazyValue) error { + return tb.WaitCtx(ctx, tokenbucket.Tokens(key.Size()+val.Len())) + } + } + + scanInternalOpts := &scanInternalOptions{ + visitPointKey: func(key *InternalKey, value LazyValue, iterInfo IteratorLevel) error { + // If the previous key is equal to the current point key, the current key was + // pinned by a snapshot. + size := uint64(key.Size()) + kind := key.Kind() + sameKey := d.equal(prevKey.UserKey, key.UserKey) + if iterInfo.Kind == IteratorLevelLSM && sameKey { + stats.Levels[iterInfo.Level].SnapshotPinnedKeys++ + stats.Levels[iterInfo.Level].SnapshotPinnedKeysBytes += size + stats.Accumulated.SnapshotPinnedKeys++ + stats.Accumulated.SnapshotPinnedKeysBytes += size + } + if iterInfo.Kind == IteratorLevelLSM { + stats.Levels[iterInfo.Level].KindsCount[kind]++ + } + if !sameKey { + if iterInfo.Kind == IteratorLevelLSM { + stats.Levels[iterInfo.Level].LatestKindsCount[kind]++ + } + stats.Accumulated.LatestKindsCount[kind]++ + } + + stats.Accumulated.KindsCount[kind]++ + prevKey.CopyFrom(*key) + stats.BytesRead += uint64(key.Size() + value.Len()) + return nil + }, + visitRangeDel: func(start, end []byte, seqNum uint64) error { + stats.Accumulated.KindsCount[InternalKeyKindRangeDelete]++ + stats.BytesRead += uint64(len(start) + len(end)) + return nil + }, + visitRangeKey: func(start, end []byte, keys []rangekey.Key) error { + stats.BytesRead += uint64(len(start) + len(end)) + for _, key := range keys { + stats.Accumulated.KindsCount[key.Kind()]++ + stats.BytesRead += uint64(len(key.Value) + len(key.Suffix)) + } + return nil + }, + includeObsoleteKeys: true, + IterOptions: IterOptions{ + KeyTypes: IterKeyTypePointsAndRanges, + LowerBound: lower, + UpperBound: upper, + }, + rateLimitFunc: rateLimitFunc, + } + iter := d.newInternalIter(snapshotIterOpts{}, scanInternalOpts) + defer iter.close() + + err := scanInternalImpl(ctx, lower, upper, iter, scanInternalOpts) + + if err != nil { + return LSMKeyStatistics{}, err + } + + return stats, nil +} + +// ObjProvider returns the objstorage.Provider for this database. Meant to be +// used for internal purposes only. +func (d *DB) ObjProvider() objstorage.Provider { + return d.objProvider +} + +func (d *DB) checkVirtualBounds(m *fileMetadata) { + if !invariants.Enabled { + return + } + + if m.HasPointKeys { + pointIter, rangeDelIter, err := d.newIters(context.TODO(), m, nil, internalIterOpts{}) + if err != nil { + panic(errors.Wrap(err, "pebble: error creating point iterator")) + } + + defer pointIter.Close() + if rangeDelIter != nil { + defer rangeDelIter.Close() + } + + pointKey, _ := pointIter.First() + var rangeDel *keyspan.Span + if rangeDelIter != nil { + rangeDel = rangeDelIter.First() + } + + // Check that the lower bound is tight. + if (rangeDel == nil || d.cmp(rangeDel.SmallestKey().UserKey, m.SmallestPointKey.UserKey) != 0) && + (pointKey == nil || d.cmp(pointKey.UserKey, m.SmallestPointKey.UserKey) != 0) { + panic(errors.Newf("pebble: virtual sstable %s lower point key bound is not tight", m.FileNum)) + } + + pointKey, _ = pointIter.Last() + rangeDel = nil + if rangeDelIter != nil { + rangeDel = rangeDelIter.Last() + } + + // Check that the upper bound is tight. + if (rangeDel == nil || d.cmp(rangeDel.LargestKey().UserKey, m.LargestPointKey.UserKey) != 0) && + (pointKey == nil || d.cmp(pointKey.UserKey, m.LargestPointKey.UserKey) != 0) { + panic(errors.Newf("pebble: virtual sstable %s upper point key bound is not tight", m.FileNum)) + } + + // Check that iterator keys are within bounds. + for key, _ := pointIter.First(); key != nil; key, _ = pointIter.Next() { + if d.cmp(key.UserKey, m.SmallestPointKey.UserKey) < 0 || d.cmp(key.UserKey, m.LargestPointKey.UserKey) > 0 { + panic(errors.Newf("pebble: virtual sstable %s point key %s is not within bounds", m.FileNum, key.UserKey)) + } + } + + if rangeDelIter != nil { + for key := rangeDelIter.First(); key != nil; key = rangeDelIter.Next() { + if d.cmp(key.SmallestKey().UserKey, m.SmallestPointKey.UserKey) < 0 { + panic(errors.Newf("pebble: virtual sstable %s point key %s is not within bounds", m.FileNum, key.SmallestKey().UserKey)) + } + + if d.cmp(key.LargestKey().UserKey, m.LargestPointKey.UserKey) > 0 { + panic(errors.Newf("pebble: virtual sstable %s point key %s is not within bounds", m.FileNum, key.LargestKey().UserKey)) + } + } + } + } + + if !m.HasRangeKeys { + return + } + + rangeKeyIter, err := d.tableNewRangeKeyIter(m, keyspan.SpanIterOptions{}) + defer rangeKeyIter.Close() + + if err != nil { + panic(errors.Wrap(err, "pebble: error creating range key iterator")) + } + + // Check that the lower bound is tight. + if d.cmp(rangeKeyIter.First().SmallestKey().UserKey, m.SmallestRangeKey.UserKey) != 0 { + panic(errors.Newf("pebble: virtual sstable %s lower range key bound is not tight", m.FileNum)) + } + + // Check that upper bound is tight. + if d.cmp(rangeKeyIter.Last().LargestKey().UserKey, m.LargestRangeKey.UserKey) != 0 { + panic(errors.Newf("pebble: virtual sstable %s upper range key bound is not tight", m.FileNum)) + } + + for key := rangeKeyIter.First(); key != nil; key = rangeKeyIter.Next() { + if d.cmp(key.SmallestKey().UserKey, m.SmallestRangeKey.UserKey) < 0 { + panic(errors.Newf("pebble: virtual sstable %s point key %s is not within bounds", m.FileNum, key.SmallestKey().UserKey)) + } + if d.cmp(key.LargestKey().UserKey, m.LargestRangeKey.UserKey) > 0 { + panic(errors.Newf("pebble: virtual sstable %s point key %s is not within bounds", m.FileNum, key.LargestKey().UserKey)) + } + } +} diff --git a/vendor/github.com/cockroachdb/pebble/event.go b/vendor/github.com/cockroachdb/pebble/event.go index f243e24..d431d07 100644 --- a/vendor/github.com/cockroachdb/pebble/event.go +++ b/vendor/github.com/cockroachdb/pebble/event.go @@ -39,10 +39,11 @@ func formatFileNums(tables []TableInfo) string { return buf.String() } -// LevelInfo contains info pertaining to a partificular level. +// LevelInfo contains info pertaining to a particular level. type LevelInfo struct { Level int Tables []TableInfo + Score float64 } func (i LevelInfo) String() string { @@ -51,8 +52,11 @@ func (i LevelInfo) String() string { // SafeFormat implements redact.SafeFormatter. func (i LevelInfo) SafeFormat(w redact.SafePrinter, _ rune) { - w.Printf("L%d [%s] (%s)", redact.Safe(i.Level), redact.Safe(formatFileNums(i.Tables)), - redact.Safe(humanize.Uint64(tablesTotalSize(i.Tables)))) + w.Printf("L%d [%s] (%s) Score=%.2f", + redact.Safe(i.Level), + redact.Safe(formatFileNums(i.Tables)), + redact.Safe(humanize.Bytes.Uint64(tablesTotalSize(i.Tables))), + redact.Safe(i.Score)) } // CompactionInfo contains the info for a compaction event. @@ -75,6 +79,27 @@ type CompactionInfo struct { TotalDuration time.Duration Done bool Err error + + SingleLevelOverlappingRatio float64 + MultiLevelOverlappingRatio float64 + + // Annotations specifies additional info to appear in a compaction's event log line + Annotations compactionAnnotations +} + +type compactionAnnotations []string + +// SafeFormat implements redact.SafeFormatter. +func (ca compactionAnnotations) SafeFormat(w redact.SafePrinter, _ rune) { + if len(ca) == 0 { + return + } + for i := range ca { + if i != 0 { + w.Print(" ") + } + w.Printf("%s", redact.SafeString(ca[i])) + } } func (i CompactionInfo) String() string { @@ -90,20 +115,25 @@ func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) { } if !i.Done { - w.Printf("[JOB %d] compacting(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason)) - w.Print(levelInfos(i.Input)) + w.Printf("[JOB %d] compacting(%s) ", + redact.Safe(i.JobID), + redact.SafeString(i.Reason)) + w.Printf("%s", i.Annotations) + w.Printf("%s; ", levelInfos(i.Input)) + w.Printf("OverlappingRatio: Single %.2f, Multi %.2f", i.SingleLevelOverlappingRatio, i.MultiLevelOverlappingRatio) return } outputSize := tablesTotalSize(i.Output.Tables) w.Printf("[JOB %d] compacted(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason)) + w.Printf("%s", i.Annotations) w.Print(levelInfos(i.Input)) w.Printf(" -> L%d [%s] (%s), in %.1fs (%.1fs total), output rate %s/s", redact.Safe(i.Output.Level), redact.Safe(formatFileNums(i.Output.Tables)), - redact.Safe(humanize.Uint64(outputSize)), + redact.Safe(humanize.Bytes.Uint64(outputSize)), redact.Safe(i.Duration.Seconds()), redact.Safe(i.TotalDuration.Seconds()), - redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds())))) + redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds())))) } type levelInfos []LevelInfo @@ -129,6 +159,9 @@ type FlushInfo struct { Reason string // Input contains the count of input memtables that were flushed. Input int + // InputBytes contains the total in-memory size of the memtable(s) that were + // flushed. This size includes skiplist indexing data structures. + InputBytes uint64 // Output contains the ouptut table generated by the flush. The output info // is empty for the flush begin event. Output []TableInfo @@ -168,7 +201,7 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) { if !i.Ingest { w.Printf("flushing %d memtable", redact.Safe(i.Input)) w.SafeString(plural) - w.Printf(" to L0") + w.Printf(" (%s) to L0", redact.Safe(humanize.Bytes.Uint64(i.InputBytes))) } else { w.Printf("flushing %d ingested table%s", redact.Safe(i.Input), plural) } @@ -180,13 +213,14 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) { if invariants.Enabled && len(i.IngestLevels) > 0 { panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) == 0")) } - w.Printf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s", + w.Printf("[JOB %d] flushed %d memtable%s (%s) to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s", redact.Safe(i.JobID), redact.Safe(i.Input), plural, + redact.Safe(humanize.Bytes.Uint64(i.InputBytes)), redact.Safe(formatFileNums(i.Output)), - redact.Safe(humanize.Uint64(outputSize)), + redact.Safe(humanize.Bytes.Uint64(outputSize)), redact.Safe(i.Duration.Seconds()), redact.Safe(i.TotalDuration.Seconds()), - redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds())))) + redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds())))) } else { if invariants.Enabled && len(i.IngestLevels) == 0 { panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) > 0")) @@ -198,12 +232,12 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) { if j > 0 { w.Printf(" +") } - w.Printf(" L%d:%s (%s)", level, redact.Safe(file.FileNum), humanize.IEC.Uint64(file.Size)) + w.Printf(" L%d:%s (%s)", level, file.FileNum, humanize.Bytes.Uint64(file.Size)) } w.Printf(" in %.1fs (%.1fs total), output rate %s/s", redact.Safe(i.Duration.Seconds()), redact.Safe(i.TotalDuration.Seconds()), - redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds())))) + redact.Safe(humanize.Bytes.Uint64(uint64(float64(outputSize)/i.Duration.Seconds())))) } } @@ -227,7 +261,7 @@ func (i ManifestCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) { w.Printf("[JOB %d] MANIFEST create error: %s", redact.Safe(i.JobID), i.Err) return } - w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum)) + w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), i.FileNum) } // ManifestDeleteInfo contains the info for a Manifest deletion event. @@ -249,7 +283,7 @@ func (i ManifestDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) { w.Printf("[JOB %d] MANIFEST delete error: %s", redact.Safe(i.JobID), i.Err) return } - w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum)) + w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), i.FileNum) } // TableCreateInfo contains the info for a table creation event. @@ -269,7 +303,7 @@ func (i TableCreateInfo) String() string { // SafeFormat implements redact.SafeFormatter. func (i TableCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) { w.Printf("[JOB %d] %s: sstable created %s", - redact.Safe(i.JobID), redact.Safe(i.Reason), redact.Safe(i.FileNum)) + redact.Safe(i.JobID), redact.Safe(i.Reason), i.FileNum) } // TableDeleteInfo contains the info for a table deletion event. @@ -288,10 +322,10 @@ func (i TableDeleteInfo) String() string { func (i TableDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) { if i.Err != nil { w.Printf("[JOB %d] sstable delete error %s: %s", - redact.Safe(i.JobID), redact.Safe(i.FileNum), i.Err) + redact.Safe(i.JobID), i.FileNum, i.Err) return } - w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum)) + w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), i.FileNum) } // TableIngestInfo contains the info for a table ingestion event. @@ -337,8 +371,8 @@ func (i TableIngestInfo) SafeFormat(w redact.SafePrinter, _ rune) { if !i.flushable { levelStr = fmt.Sprintf("L%d:", t.Level) } - w.Printf(" %s%s (%s)", redact.Safe(levelStr), redact.Safe(t.FileNum), - redact.Safe(humanize.Uint64(t.Size))) + w.Printf(" %s%s (%s)", redact.Safe(levelStr), t.FileNum, + redact.Safe(humanize.Bytes.Uint64(t.Size))) } } @@ -399,12 +433,12 @@ func (i WALCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) { } if i.RecycledFileNum == 0 { - w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum)) + w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), i.FileNum) return } w.Printf("[JOB %d] WAL created %s (recycled %s)", - redact.Safe(i.JobID), redact.Safe(i.FileNum), redact.Safe(i.RecycledFileNum)) + redact.Safe(i.JobID), i.FileNum, i.RecycledFileNum) } // WALDeleteInfo contains the info for a WAL deletion event. @@ -426,7 +460,7 @@ func (i WALDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) { w.Printf("[JOB %d] WAL delete error: %s", redact.Safe(i.JobID), i.Err) return } - w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum)) + w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), i.FileNum) } // WriteStallBeginInfo contains the info for a write stall begin event. diff --git a/vendor/github.com/cockroachdb/pebble/external_iterator.go b/vendor/github.com/cockroachdb/pebble/external_iterator.go index 3c45c2c..1bd00c3 100644 --- a/vendor/github.com/cockroachdb/pebble/external_iterator.go +++ b/vendor/github.com/cockroachdb/pebble/external_iterator.go @@ -209,15 +209,15 @@ func createExternalPointIter(ctx context.Context, it *Iterator) (internalIterato pointIter internalIterator err error ) - pointIter, err = r.NewIterWithBlockPropertyFiltersAndContext( - ctx, - it.opts.LowerBound, - it.opts.UpperBound, - nil, /* BlockPropertiesFilterer */ - false, /* useFilterBlock */ - &it.stats.InternalStats, - sstable.TrivialReaderProvider{Reader: r}, - ) + // We could set hideObsoletePoints=true, since we are reading at + // InternalKeySeqNumMax, but we don't bother since these sstables should + // not have obsolete points (so the performance optimization is + // unnecessary), and we don't want to bother constructing a + // BlockPropertiesFilterer that includes obsoleteKeyBlockPropertyFilter. + pointIter, err = r.NewIterWithBlockPropertyFiltersAndContextEtc( + ctx, it.opts.LowerBound, it.opts.UpperBound, nil, /* BlockPropertiesFilterer */ + false /* hideObsoletePoints */, false, /* useFilterBlock */ + &it.stats.InternalStats, sstable.TrivialReaderProvider{Reader: r}) if err != nil { return nil, err } @@ -311,7 +311,7 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) { base.InternalKeySeqNumMax, it.opts.LowerBound, it.opts.UpperBound, &it.hasPrefix, &it.prefixOrFullSeekKey, - true /* onlySets */, &it.rangeKey.internal, + false /* internalKeys */, &it.rangeKey.internal, ) for i := range rangeKeyIters { it.rangeKey.iterConfig.AddLevel(rangeKeyIters[i]) @@ -319,8 +319,12 @@ func finishInitializingExternal(ctx context.Context, it *Iterator) { } } if it.rangeKey != nil { - it.rangeKey.iiter.Init(&it.comparer, it.iter, it.rangeKey.rangeKeyIter, &it.rangeKeyMasking, - it.opts.LowerBound, it.opts.UpperBound) + it.rangeKey.iiter.Init(&it.comparer, it.iter, it.rangeKey.rangeKeyIter, + keyspan.InterleavingIterOpts{ + Mask: &it.rangeKeyMasking, + LowerBound: it.opts.LowerBound, + UpperBound: it.opts.UpperBound, + }) it.iter = &it.rangeKey.iiter } } diff --git a/vendor/github.com/cockroachdb/pebble/filenames.go b/vendor/github.com/cockroachdb/pebble/filenames.go index bc33b38..07d74c8 100644 --- a/vendor/github.com/cockroachdb/pebble/filenames.go +++ b/vendor/github.com/cockroachdb/pebble/filenames.go @@ -33,7 +33,7 @@ const ( // NB: This is a low-level routine and typically not what you want to // use. Newer versions of Pebble running newer format major versions do // not use the CURRENT file. See setCurrentFunc in version_set.go. -func setCurrentFile(dirname string, fs vfs.FS, fileNum FileNum) error { +func setCurrentFile(dirname string, fs vfs.FS, fileNum base.DiskFileNum) error { newFilename := base.MakeFilepath(fs, dirname, fileTypeCurrent, fileNum) oldFilename := base.MakeFilepath(fs, dirname, fileTypeTemp, fileNum) fs.Remove(oldFilename) diff --git a/vendor/github.com/cockroachdb/pebble/flushable.go b/vendor/github.com/cockroachdb/pebble/flushable.go index 13fca5f..09abee3 100644 --- a/vendor/github.com/cockroachdb/pebble/flushable.go +++ b/vendor/github.com/cockroachdb/pebble/flushable.go @@ -60,20 +60,20 @@ type flushableEntry struct { // flushable is a memTable, when the reader refs drops to zero, the writer // refs will already be zero because the memtable will have been flushed and // that only occurs once the writer refs drops to zero. - readerRefs int32 + readerRefs atomic.Int32 // Closure to invoke to release memory accounting. releaseMemAccounting func() // unrefFiles, if not nil, should be invoked to decrease the ref count of // files which are backing the flushable. - unrefFiles func() []*fileMetadata + unrefFiles func() []*fileBacking // deleteFnLocked should be called if the caller is holding DB.mu. - deleteFnLocked func(obsolete []*fileMetadata) + deleteFnLocked func(obsolete []*fileBacking) // deleteFn should be called if the caller is not holding DB.mu. - deleteFn func(obsolete []*fileMetadata) + deleteFn func(obsolete []*fileBacking) } func (e *flushableEntry) readerRef() { - switch v := atomic.AddInt32(&e.readerRefs, 1); { + switch v := e.readerRefs.Add(1); { case v <= 1: panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v)) } @@ -90,9 +90,9 @@ func (e *flushableEntry) readerUnrefLocked(deleteFiles bool) { } func (e *flushableEntry) readerUnrefHelper( - deleteFiles bool, deleteFn func(obsolete []*manifest.FileMetadata), + deleteFiles bool, deleteFn func(obsolete []*fileBacking), ) { - switch v := atomic.AddInt32(&e.readerRefs, -1); { + switch v := e.readerRefs.Add(-1); { case v < 0: panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v)) case v == 0: @@ -116,9 +116,8 @@ type flushableList []*flushableEntry // ingestedFlushable is the implementation of the flushable interface for the // ingesting sstables which are added to the flushable list. type ingestedFlushable struct { - files []*fileMetadata - cmp Compare - split Split + files []physicalMeta + comparer *Comparer newIters tableNewIters newRangeKeyIters keyspan.TableNewSpanIter @@ -131,26 +130,27 @@ type ingestedFlushable struct { func newIngestedFlushable( files []*fileMetadata, - cmp Compare, - split Split, + comparer *Comparer, newIters tableNewIters, newRangeKeyIters keyspan.TableNewSpanIter, ) *ingestedFlushable { + var physicalFiles []physicalMeta + var hasRangeKeys bool + for _, f := range files { + if f.HasRangeKeys { + hasRangeKeys = true + } + physicalFiles = append(physicalFiles, f.PhysicalMeta()) + } + ret := &ingestedFlushable{ - files: files, - cmp: cmp, - split: split, + files: physicalFiles, + comparer: comparer, newIters: newIters, newRangeKeyIters: newRangeKeyIters, // slice is immutable and can be set once and used many times. - slice: manifest.NewLevelSliceKeySorted(cmp, files), - } - - for _, f := range files { - if f.HasRangeKeys { - ret.hasRangeKeys = true - break - } + slice: manifest.NewLevelSliceKeySorted(comparer.Compare, files), + hasRangeKeys: hasRangeKeys, } return ret @@ -159,6 +159,7 @@ func newIngestedFlushable( // TODO(sumeer): ingestedFlushable iters also need to plumb context for // tracing. +// newIter is part of the flushable interface. func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator { var opts IterOptions if o != nil { @@ -169,10 +170,11 @@ func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator { // aren't truly levels in the lsm. Right now, the encoding only supports // L0 sublevels, and the rest of the levels in the lsm. return newLevelIter( - opts, s.cmp, s.split, s.newIters, s.slice.Iter(), manifest.Level(0), nil, + opts, s.comparer, s.newIters, s.slice.Iter(), manifest.Level(0), internalIterOpts{}, ) } +// newFlushIter is part of the flushable interface. func (s *ingestedFlushable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator { // newFlushIter is only used for writing memtables to disk as sstables. // Since ingested sstables are already present on disk, they don't need to @@ -181,7 +183,7 @@ func (s *ingestedFlushable) newFlushIter(o *IterOptions, bytesFlushed *uint64) i } func (s *ingestedFlushable) constructRangeDelIter( - file *manifest.FileMetadata, _ *keyspan.SpanIterOptions, + file *manifest.FileMetadata, _ keyspan.SpanIterOptions, ) (keyspan.FragmentIterator, error) { // Note that the keyspan level iter expects a non-nil iterator to be // returned even if there is an error. So, we return the emptyKeyspanIter. @@ -196,41 +198,47 @@ func (s *ingestedFlushable) constructRangeDelIter( return rangeDelIter, nil } +// newRangeDelIter is part of the flushable interface. // TODO(bananabrick): Using a level iter instead of a keyspan level iter to // surface range deletes is more efficient. func (s *ingestedFlushable) newRangeDelIter(_ *IterOptions) keyspan.FragmentIterator { return keyspan.NewLevelIter( - keyspan.SpanIterOptions{}, s.cmp, + keyspan.SpanIterOptions{}, s.comparer.Compare, s.constructRangeDelIter, s.slice.Iter(), manifest.Level(0), manifest.KeyTypePoint, ) } +// newRangeKeyIter is part of the flushable interface. func (s *ingestedFlushable) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator { if !s.containsRangeKeys() { return nil } return keyspan.NewLevelIter( - keyspan.SpanIterOptions{}, s.cmp, s.newRangeKeyIters, + keyspan.SpanIterOptions{}, s.comparer.Compare, s.newRangeKeyIters, s.slice.Iter(), manifest.Level(0), manifest.KeyTypeRange, ) } +// containsRangeKeys is part of the flushable interface. func (s *ingestedFlushable) containsRangeKeys() bool { return s.hasRangeKeys } +// inuseBytes is part of the flushable interface. func (s *ingestedFlushable) inuseBytes() uint64 { // inuseBytes is only used when memtables are flushed to disk as sstables. panic("pebble: not implemented") } +// totalBytes is part of the flushable interface. func (s *ingestedFlushable) totalBytes() uint64 { // We don't allocate additional bytes for the ingestedFlushable. return 0 } +// readyForFlush is part of the flushable interface. func (s *ingestedFlushable) readyForFlush() bool { // ingestedFlushable should always be ready to flush. However, note that // memtables before the ingested sstables in the memtable queue must be diff --git a/vendor/github.com/cockroachdb/pebble/format_major_version.go b/vendor/github.com/cockroachdb/pebble/format_major_version.go index 9e96709..45f1bc9 100644 --- a/vendor/github.com/cockroachdb/pebble/format_major_version.go +++ b/vendor/github.com/cockroachdb/pebble/format_major_version.go @@ -28,10 +28,13 @@ import ( // FormatVersion that the default corresponds to may change with time. type FormatMajorVersion uint64 +// SafeValue implements redact.SafeValue. +func (v FormatMajorVersion) SafeValue() {} + // String implements fmt.Stringer. func (v FormatMajorVersion) String() string { - // NB: This must not change. It's used as the value for the the - // on-disk version marker file. + // NB: This must not change. It's used as the value for the on-disk + // version marker file. // // Specifically, this value must always parse as a base 10 integer // that fits in a uint64. We format it as zero-padded, 3-digit @@ -110,12 +113,12 @@ const ( // 23.1 versions. - // FormatUnusedPrePebblev1MarkedCompacted is an unused format major version. + // formatUnusedPrePebblev1MarkedCompacted is an unused format major version. // This format major version was originally intended to ship in the 23.1 // release. It was later decided that this should be deferred until a // subsequent release. The original ordering is preserved so as not to // introduce breaking changes in Cockroach. - FormatUnusedPrePebblev1MarkedCompacted + formatUnusedPrePebblev1MarkedCompacted // FormatSSTableValueBlocks is a format major version that adds support for // storing values in value blocks in the sstable. Value block support is not @@ -151,8 +154,28 @@ const ( // compactions for files marked for compaction are complete. FormatPrePebblev1MarkedCompacted + // FormatDeleteSizedAndObsolete is a format major version that adds support + // for deletion tombstones that encode the size of the value they're + // expected to delete. This format major version is required before the + // associated key kind may be committed through batch applications or + // ingests. It also adds support for keys that are marked obsolete (see + // sstable/format.go for details). + FormatDeleteSizedAndObsolete + + // FormatVirtualSSTables is a format major version that adds support for + // virtual sstables that can reference a sub-range of keys in an underlying + // physical sstable. This information is persisted through new, + // backward-incompatible fields in the Manifest, and therefore requires + // a format major version. + FormatVirtualSSTables + + // internalFormatNewest holds the newest format major version, including + // experimental ones excluded from the exported FormatNewest constant until + // they've stabilized. Used in tests. + internalFormatNewest FormatMajorVersion = iota - 1 + // FormatNewest always contains the most recent format major version. - FormatNewest FormatMajorVersion = iota - 1 + FormatNewest FormatMajorVersion = internalFormatNewest ) // MaxTableFormat returns the maximum sstable.TableFormat that can be used at @@ -166,11 +189,12 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat { FormatSplitUserKeysMarkedCompacted: return sstable.TableFormatPebblev1 case FormatRangeKeys, FormatMinTableFormatPebblev1, FormatPrePebblev1Marked, - FormatUnusedPrePebblev1MarkedCompacted: + formatUnusedPrePebblev1MarkedCompacted: return sstable.TableFormatPebblev2 - case FormatSSTableValueBlocks, FormatFlushableIngest, - FormatPrePebblev1MarkedCompacted: + case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted: return sstable.TableFormatPebblev3 + case FormatDeleteSizedAndObsolete, FormatVirtualSSTables: + return sstable.TableFormatPebblev4 default: panic(fmt.Sprintf("pebble: unsupported format major version: %s", v)) } @@ -186,14 +210,27 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat { FormatRangeKeys: return sstable.TableFormatLevelDB case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked, - FormatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks, - FormatFlushableIngest, FormatPrePebblev1MarkedCompacted: + formatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks, + FormatFlushableIngest, FormatPrePebblev1MarkedCompacted, + FormatDeleteSizedAndObsolete, FormatVirtualSSTables: return sstable.TableFormatPebblev1 default: panic(fmt.Sprintf("pebble: unsupported format major version: %s", v)) } } +// orderingInvariants returns an enum encoding the set of invariants that must +// hold within the receiver format major version. Invariants only get stricter +// as the format major version advances, so it is okay to retrieve the +// invariants from the current format major version and by the time the +// invariants are enforced, the format major version has advanced. +func (v FormatMajorVersion) orderingInvariants() manifest.OrderingInvariants { + if v < FormatSplitUserKeysMarkedCompacted { + return manifest.AllowSplitUserKeys + } + return manifest.ProhibitSplitUserKeys +} + // formatMajorVersionMigrations defines the migrations from one format // major version to the next. Each migration is defined as a closure // which will be invoked on the database before the new format major @@ -219,7 +256,7 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{ // guaranteed to exist, because we unconditionally locate it // during Open. manifestFileNum := d.mu.versions.manifestFileNum - filename := base.MakeFilename(fileTypeManifest, manifestFileNum) + filename := base.MakeFilename(fileTypeManifest, manifestFileNum.DiskFileNum()) if err := d.mu.versions.manifestMarker.Move(filename); err != nil { return errors.Wrap(err, "moving manifest marker") } @@ -257,7 +294,7 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{ // version that does not know about format major versions // attempts to open the database, it will error avoiding // accidental corruption. - if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, 0); err != nil { + if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, base.FileNum(0).DiskFileNum()); err != nil { return err } return d.finalizeFormatVersUpgrade(FormatVersioned) @@ -301,9 +338,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{ } return d.finalizeFormatVersUpgrade(FormatPrePebblev1Marked) }, - FormatUnusedPrePebblev1MarkedCompacted: func(d *DB) error { + formatUnusedPrePebblev1MarkedCompacted: func(d *DB) error { // Intentional no-op. - return d.finalizeFormatVersUpgrade(FormatUnusedPrePebblev1MarkedCompacted) + return d.finalizeFormatVersUpgrade(formatUnusedPrePebblev1MarkedCompacted) }, FormatSSTableValueBlocks: func(d *DB) error { return d.finalizeFormatVersUpgrade(FormatSSTableValueBlocks) @@ -320,6 +357,12 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{ } return d.finalizeFormatVersUpgrade(FormatPrePebblev1MarkedCompacted) }, + FormatDeleteSizedAndObsolete: func(d *DB) error { + return d.finalizeFormatVersUpgrade(FormatDeleteSizedAndObsolete) + }, + FormatVirtualSSTables: func(d *DB) error { + return d.finalizeFormatVersUpgrade(FormatVirtualSSTables) + }, } const formatVersionMarkerName = `format-version` @@ -342,7 +385,7 @@ func lookupFormatMajorVersion( if vers == FormatDefault { return 0, nil, errors.Newf("pebble: default format major version should not persisted", vers) } - if vers > FormatNewest { + if vers > internalFormatNewest { return 0, nil, errors.Newf("pebble: database %q written in format major version %d", dirname, vers) } return vers, m, nil @@ -376,8 +419,8 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro if d.opts.ReadOnly { return ErrReadOnly } - if formatVers > FormatNewest { - // Guard against accidentally forgetting to update FormatNewest. + if formatVers > internalFormatNewest { + // Guard against accidentally forgetting to update internalFormatNewest. return errors.Errorf("pebble: unknown format version %d", formatVers) } if currentVers := d.FormatMajorVersion(); currentVers > formatVers { @@ -519,17 +562,24 @@ var markFilesPrePebblev1 = func(tc *tableCacheContainer) findFilesFunc { for l := numLevels - 1; l > 0; l-- { iter := v.Levels[l].Iter() for f := iter.First(); f != nil; f = iter.Next() { - err = tc.withReader(f, func(r *sstable.Reader) error { - tf, err := r.TableFormat() - if err != nil { - return err - } - if tf < sstable.TableFormatPebblev1 { - found = true - files[l] = append(files[l], f) - } - return nil - }) + if f.Virtual { + // Any physical sstable which has been virtualized must + // have already undergone this migration, and we don't + // need to worry about the virtual sstable themselves. + panic("pebble: unexpected virtual sstable during migration") + } + err = tc.withReader( + f.PhysicalMeta(), func(r *sstable.Reader) error { + tf, err := r.TableFormat() + if err != nil { + return err + } + if tf < sstable.TableFormatPebblev1 { + found = true + files[l] = append(files[l], f) + } + return nil + }) if err != nil { return } diff --git a/vendor/github.com/cockroachdb/pebble/get_iter.go b/vendor/github.com/cockroachdb/pebble/get_iter.go index f1fc25c..99c5d7c 100644 --- a/vendor/github.com/cockroachdb/pebble/get_iter.go +++ b/vendor/github.com/cockroachdb/pebble/get_iter.go @@ -19,8 +19,7 @@ import ( // lazily. type getIter struct { logger Logger - cmp Compare - equal Equal + comparer *Comparer newIters tableNewIters snapshot uint64 key []byte @@ -83,7 +82,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) { // key. Every call to levelIter.Next() potentially switches to a new // table and thus reinitializes rangeDelIter. if g.rangeDelIter != nil { - g.tombstone = keyspan.Get(g.cmp, g.rangeDelIter, g.key) + g.tombstone = keyspan.Get(g.comparer.Compare, g.rangeDelIter, g.key) if g.err = g.rangeDelIter.Close(); g.err != nil { return nil, base.LazyValue{} } @@ -101,7 +100,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) { g.iter = nil return nil, base.LazyValue{} } - if g.equal(g.key, key.UserKey) { + if g.comparer.Equal(g.key, key.UserKey) { if !key.Visible(g.snapshot, base.InternalKeySeqNumMax) { g.iterKey, g.iterValue = g.iter.Next() continue @@ -158,12 +157,25 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) { if n := len(g.l0); n > 0 { files := g.l0[n-1].Iter() g.l0 = g.l0[:n-1] - iterOpts := IterOptions{logger: g.logger} - g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters, + iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot} + g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters, files, manifest.L0Sublevel(n), internalIterOpts{}) g.levelIter.initRangeDel(&g.rangeDelIter) + bc := levelIterBoundaryContext{} + g.levelIter.initBoundaryContext(&bc) g.iter = &g.levelIter - g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone) + + // Compute the key prefix for bloom filtering if split function is + // specified, or use the user key as default. + prefix := g.key + if g.comparer.Split != nil { + prefix = g.key[:g.comparer.Split(g.key)] + } + g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone) + if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey { + g.iterKey = nil + g.iterValue = base.LazyValue{} + } continue } g.level++ @@ -177,13 +189,26 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) { continue } - iterOpts := IterOptions{logger: g.logger} - g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters, + iterOpts := IterOptions{logger: g.logger, snapshotForHideObsoletePoints: g.snapshot} + g.levelIter.init(context.Background(), iterOpts, g.comparer, g.newIters, g.version.Levels[g.level].Iter(), manifest.Level(g.level), internalIterOpts{}) g.levelIter.initRangeDel(&g.rangeDelIter) + bc := levelIterBoundaryContext{} + g.levelIter.initBoundaryContext(&bc) g.level++ g.iter = &g.levelIter - g.iterKey, g.iterValue = g.iter.SeekGE(g.key, base.SeekGEFlagsNone) + + // Compute the key prefix for bloom filtering if split function is + // specified, or use the user key as default. + prefix := g.key + if g.comparer.Split != nil { + prefix = g.key[:g.comparer.Split(g.key)] + } + g.iterKey, g.iterValue = g.iter.SeekPrefixGE(prefix, g.key, base.SeekGEFlagsNone) + if bc.isSyntheticIterBoundsKey || bc.isIgnorableBoundaryKey { + g.iterKey = nil + g.iterValue = base.LazyValue{} + } } } diff --git a/vendor/github.com/cockroachdb/pebble/ingest.go b/vendor/github.com/cockroachdb/pebble/ingest.go index bd4b34b..fa6304a 100644 --- a/vendor/github.com/cockroachdb/pebble/ingest.go +++ b/vendor/github.com/cockroachdb/pebble/ingest.go @@ -7,15 +7,16 @@ package pebble import ( "context" "sort" - "sync/atomic" "time" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/internal/private" "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/remote" "github.com/cockroachdb/pebble/sstable" ) @@ -34,6 +35,47 @@ func sstableKeyCompare(userCmp Compare, a, b InternalKey) int { return 0 } +// KeyRange encodes a key range in user key space. A KeyRange's Start is +// inclusive while its End is exclusive. +type KeyRange struct { + Start, End []byte +} + +// Valid returns true if the KeyRange is defined. +func (k *KeyRange) Valid() bool { + return k.Start != nil && k.End != nil +} + +// Contains returns whether the specified key exists in the KeyRange. +func (k *KeyRange) Contains(cmp base.Compare, key InternalKey) bool { + v := cmp(key.UserKey, k.End) + return (v < 0 || (v == 0 && key.IsExclusiveSentinel())) && cmp(k.Start, key.UserKey) <= 0 +} + +// OverlapsInternalKeyRange checks if the specified internal key range has an +// overlap with the KeyRange. Note that we aren't checking for full containment +// of smallest-largest within k, rather just that there's some intersection +// between the two ranges. +func (k *KeyRange) OverlapsInternalKeyRange(cmp base.Compare, smallest, largest InternalKey) bool { + v := cmp(k.Start, largest.UserKey) + return v <= 0 && !(largest.IsExclusiveSentinel() && v == 0) && + cmp(k.End, smallest.UserKey) > 0 +} + +// Overlaps checks if the specified file has an overlap with the KeyRange. +// Note that we aren't checking for full containment of m within k, rather just +// that there's some intersection between m and k's bounds. +func (k *KeyRange) Overlaps(cmp base.Compare, m *fileMetadata) bool { + return k.OverlapsInternalKeyRange(cmp, m.Smallest, m.Largest) +} + +// OverlapsKeyRange checks if this span overlaps with the provided KeyRange. +// Note that we aren't checking for full containment of either span in the other, +// just that there's a key x that is in both key ranges. +func (k *KeyRange) OverlapsKeyRange(cmp Compare, span KeyRange) bool { + return cmp(k.Start, span.End) < 0 && cmp(k.End, span.Start) > 0 +} + func ingestValidateKey(opts *Options, key *InternalKey) error { if key.Kind() == InternalKeyKindInvalid { return base.CorruptionErrorf("pebble: external sstable has corrupted key: %s", @@ -46,19 +88,143 @@ func ingestValidateKey(opts *Options, key *InternalKey) error { return nil } -func ingestLoad1( - opts *Options, fmv FormatMajorVersion, path string, cacheID uint64, fileNum FileNum, +// ingestSynthesizeShared constructs a fileMetadata for one shared sstable owned +// or shared by another node. +func ingestSynthesizeShared( + opts *Options, sm SharedSSTMeta, fileNum base.DiskFileNum, ) (*fileMetadata, error) { - f, err := opts.FS.Open(path) + if sm.Size == 0 { + // Disallow 0 file sizes + return nil, errors.New("pebble: cannot ingest shared file with size 0") + } + // Don't load table stats. Doing a round trip to shared storage, one SST + // at a time is not worth it as it slows down ingestion. + meta := &fileMetadata{ + FileNum: fileNum.FileNum(), + CreationTime: time.Now().Unix(), + Virtual: true, + Size: sm.Size, + } + meta.InitProviderBacking(fileNum) + // Set the underlying FileBacking's size to the same size as the virtualized + // view of the sstable. This ensures that we don't over-prioritize this + // sstable for compaction just yet, as we do not have a clear sense of what + // parts of this sstable are referenced by other nodes. + meta.FileBacking.Size = sm.Size + if sm.LargestRangeKey.Valid() && sm.LargestRangeKey.UserKey != nil { + // Initialize meta.{HasRangeKeys,Smallest,Largest}, etc. + // + // NB: We create new internal keys and pass them into ExternalRangeKeyBounds + // so that we can sub a zero sequence number into the bounds. We can set + // the sequence number to anything here; it'll be reset in ingestUpdateSeqNum + // anyway. However we do need to use the same sequence number across all + // bound keys at this step so that we end up with bounds that are consistent + // across point/range keys. + smallestRangeKey := base.MakeInternalKey(sm.SmallestRangeKey.UserKey, 0, sm.SmallestRangeKey.Kind()) + largestRangeKey := base.MakeExclusiveSentinelKey(sm.LargestRangeKey.Kind(), sm.LargestRangeKey.UserKey) + meta.ExtendRangeKeyBounds(opts.Comparer.Compare, smallestRangeKey, largestRangeKey) + } + if sm.LargestPointKey.Valid() && sm.LargestPointKey.UserKey != nil { + // Initialize meta.{HasPointKeys,Smallest,Largest}, etc. + // + // See point above in the ExtendRangeKeyBounds call on why we use a zero + // sequence number here. + smallestPointKey := base.MakeInternalKey(sm.SmallestPointKey.UserKey, 0, sm.SmallestPointKey.Kind()) + largestPointKey := base.MakeInternalKey(sm.LargestPointKey.UserKey, 0, sm.LargestPointKey.Kind()) + if sm.LargestPointKey.IsExclusiveSentinel() { + largestPointKey = base.MakeRangeDeleteSentinelKey(sm.LargestPointKey.UserKey) + } + meta.ExtendPointKeyBounds(opts.Comparer.Compare, smallestPointKey, largestPointKey) + } + if err := meta.Validate(opts.Comparer.Compare, opts.Comparer.FormatKey); err != nil { + return nil, err + } + return meta, nil +} + +// ingestLoad1External loads the fileMetadata for one external sstable. +// Sequence number and target level calculation happens during prepare/apply. +func ingestLoad1External( + opts *Options, + e ExternalFile, + fileNum base.DiskFileNum, + objprovider objstorage.Provider, + jobID int, +) (*fileMetadata, error) { + if e.Size == 0 { + // Disallow 0 file sizes + return nil, errors.New("pebble: cannot ingest external file with size 0") + } + if !e.HasRangeKey && !e.HasPointKey { + return nil, errors.New("pebble: cannot ingest external file with no point or range keys") + } + // Don't load table stats. Doing a round trip to shared storage, one SST + // at a time is not worth it as it slows down ingestion. + meta := &fileMetadata{} + meta.FileNum = fileNum.FileNum() + meta.CreationTime = time.Now().Unix() + meta.Virtual = true + meta.Size = e.Size + meta.InitProviderBacking(fileNum) + + // Try to resolve a reference to the external file. + backing, err := objprovider.CreateExternalObjectBacking(e.Locator, e.ObjName) if err != nil { return nil, err } - - readable, err := sstable.NewSimpleReadable(f) + metas, err := objprovider.AttachRemoteObjects([]objstorage.RemoteObjectToAttach{{ + FileNum: fileNum, + FileType: fileTypeTable, + Backing: backing, + }}) if err != nil { return nil, err } + if opts.EventListener.TableCreated != nil { + opts.EventListener.TableCreated(TableCreateInfo{ + JobID: jobID, + Reason: "ingesting", + Path: objprovider.Path(metas[0]), + FileNum: fileNum.FileNum(), + }) + } + // In the name of keeping this ingestion as fast as possible, we avoid + // *all* existence checks and synthesize a file metadata with smallest/largest + // keys that overlap whatever the passed-in span was. + smallestCopy := make([]byte, len(e.SmallestUserKey)) + copy(smallestCopy, e.SmallestUserKey) + largestCopy := make([]byte, len(e.LargestUserKey)) + copy(largestCopy, e.LargestUserKey) + if e.HasPointKey { + meta.ExtendPointKeyBounds(opts.Comparer.Compare, base.MakeInternalKey(smallestCopy, 0, InternalKeyKindMax), + base.MakeRangeDeleteSentinelKey(largestCopy)) + } + if e.HasRangeKey { + meta.ExtendRangeKeyBounds(opts.Comparer.Compare, base.MakeInternalKey(smallestCopy, 0, InternalKeyKindRangeKeySet), + base.MakeExclusiveSentinelKey(InternalKeyKindRangeKeyDelete, largestCopy)) + } + // Set the underlying FileBacking's size to the same size as the virtualized + // view of the sstable. This ensures that we don't over-prioritize this + // sstable for compaction just yet, as we do not have a clear sense of + // what parts of this sstable are referenced by other nodes. + meta.FileBacking.Size = e.Size + + if err := meta.Validate(opts.Comparer.Compare, opts.Comparer.FormatKey); err != nil { + return nil, err + } + return meta, nil +} + +// ingestLoad1 creates the FileMetadata for one file. This file will be owned +// by this store. +func ingestLoad1( + opts *Options, + fmv FormatMajorVersion, + readable objstorage.Readable, + cacheID uint64, + fileNum base.DiskFileNum, +) (*fileMetadata, error) { cacheOpts := private.SSTableCacheOpts(cacheID, fileNum).(sstable.ReaderOption) r, err := sstable.NewReader(readable, opts.MakeReaderOptions(), cacheOpts) if err != nil { @@ -79,9 +245,10 @@ func ingestLoad1( } meta := &fileMetadata{} - meta.FileNum = fileNum + meta.FileNum = fileNum.FileNum() meta.Size = uint64(readable.Size()) meta.CreationTime = time.Now().Unix() + meta.InitPhysicalBacking() // Avoid loading into the table cache for collecting stats if we // don't need to. If there are no range deletions, we have all the @@ -92,7 +259,7 @@ func ingestLoad1( // disallowing removal of an open file. Under MemFS, if we don't populate // meta.Stats here, the file will be loaded into the table cache for // calculating stats before we can remove the original link. - maybeSetStatsFromProperties(meta, &r.Properties) + maybeSetStatsFromProperties(meta.PhysicalMeta(), &r.Properties) { iter, err := r.NewIter(nil /* lower */, nil /* upper */) @@ -195,22 +362,83 @@ func ingestLoad1( return meta, nil } +type ingestLoadResult struct { + localMeta, sharedMeta []*fileMetadata + externalMeta []*fileMetadata + localPaths []string + sharedLevels []uint8 + fileCount int +} + func ingestLoad( - opts *Options, fmv FormatMajorVersion, paths []string, cacheID uint64, pending []FileNum, -) ([]*fileMetadata, []string, error) { + opts *Options, + fmv FormatMajorVersion, + paths []string, + shared []SharedSSTMeta, + external []ExternalFile, + cacheID uint64, + pending []base.DiskFileNum, + objProvider objstorage.Provider, + jobID int, +) (ingestLoadResult, error) { meta := make([]*fileMetadata, 0, len(paths)) newPaths := make([]string, 0, len(paths)) for i := range paths { - m, err := ingestLoad1(opts, fmv, paths[i], cacheID, pending[i]) + f, err := opts.FS.Open(paths[i]) if err != nil { - return nil, nil, err + return ingestLoadResult{}, err + } + + readable, err := sstable.NewSimpleReadable(f) + if err != nil { + return ingestLoadResult{}, err + } + m, err := ingestLoad1(opts, fmv, readable, cacheID, pending[i]) + if err != nil { + return ingestLoadResult{}, err } if m != nil { meta = append(meta, m) newPaths = append(newPaths, paths[i]) } } - return meta, newPaths, nil + if len(shared) == 0 && len(external) == 0 { + return ingestLoadResult{localMeta: meta, localPaths: newPaths, fileCount: len(meta)}, nil + } + + // Sort the shared files according to level. + sort.Sort(sharedByLevel(shared)) + + sharedMeta := make([]*fileMetadata, 0, len(shared)) + levels := make([]uint8, 0, len(shared)) + for i := range shared { + m, err := ingestSynthesizeShared(opts, shared[i], pending[len(paths)+i]) + if err != nil { + return ingestLoadResult{}, err + } + if shared[i].Level < sharedLevelsStart { + return ingestLoadResult{}, errors.New("cannot ingest shared file in level below sharedLevelsStart") + } + sharedMeta = append(sharedMeta, m) + levels = append(levels, shared[i].Level) + } + externalMeta := make([]*fileMetadata, 0, len(external)) + for i := range external { + m, err := ingestLoad1External(opts, external[i], pending[len(paths)+len(shared)+i], objProvider, jobID) + if err != nil { + return ingestLoadResult{}, err + } + externalMeta = append(externalMeta, m) + } + result := ingestLoadResult{ + localMeta: meta, + sharedMeta: sharedMeta, + externalMeta: externalMeta, + localPaths: newPaths, + sharedLevels: levels, + fileCount: len(meta) + len(sharedMeta) + len(externalMeta), + } + return result, nil } // Struct for sorting metadatas by smallest user keys, while ensuring the @@ -232,32 +460,79 @@ func (m metaAndPaths) Less(i, j int) bool { func (m metaAndPaths) Swap(i, j int) { m.meta[i], m.meta[j] = m.meta[j], m.meta[i] - m.paths[i], m.paths[j] = m.paths[j], m.paths[i] + if m.paths != nil { + m.paths[i], m.paths[j] = m.paths[j], m.paths[i] + } } -func ingestSortAndVerify(cmp Compare, meta []*fileMetadata, paths []string) error { - if len(meta) <= 1 { +func ingestSortAndVerify(cmp Compare, lr ingestLoadResult, exciseSpan KeyRange) error { + // Verify that all the shared files (i.e. files in sharedMeta) + // fit within the exciseSpan. + for i := range lr.sharedMeta { + f := lr.sharedMeta[i] + if !exciseSpan.Contains(cmp, f.Smallest) || !exciseSpan.Contains(cmp, f.Largest) { + return errors.AssertionFailedf("pebble: shared file outside of excise span, span [%s-%s), file = %s", exciseSpan.Start, exciseSpan.End, f.String()) + } + } + if len(lr.externalMeta) > 0 { + if len(lr.localMeta) > 0 || len(lr.sharedMeta) > 0 { + // Currently we only support external ingests on their own. If external + // files are present alongside local/shared files, return an error. + return errors.AssertionFailedf("pebble: external files cannot be ingested atomically alongside other types of files") + } + sort.Sort(&metaAndPaths{ + meta: lr.externalMeta, + cmp: cmp, + }) + for i := 1; i < len(lr.externalMeta); i++ { + if sstableKeyCompare(cmp, lr.externalMeta[i-1].Largest, lr.externalMeta[i].Smallest) >= 0 { + return errors.AssertionFailedf("pebble: external sstables have overlapping ranges") + } + } + return nil + } + if len(lr.localMeta) <= 1 || len(lr.localPaths) <= 1 { return nil } sort.Sort(&metaAndPaths{ - meta: meta, - paths: paths, + meta: lr.localMeta, + paths: lr.localPaths, cmp: cmp, }) - for i := 1; i < len(meta); i++ { - if sstableKeyCompare(cmp, meta[i-1].Largest, meta[i].Smallest) >= 0 { - return errors.New("pebble: external sstables have overlapping ranges") + for i := 1; i < len(lr.localPaths); i++ { + if sstableKeyCompare(cmp, lr.localMeta[i-1].Largest, lr.localMeta[i].Smallest) >= 0 { + return errors.AssertionFailedf("pebble: local ingestion sstables have overlapping ranges") + } + } + if len(lr.sharedMeta) == 0 { + return nil + } + filesInLevel := make([]*fileMetadata, 0, len(lr.sharedMeta)) + for l := sharedLevelsStart; l < numLevels; l++ { + filesInLevel = filesInLevel[:0] + for i := range lr.sharedMeta { + if lr.sharedLevels[i] == uint8(l) { + filesInLevel = append(filesInLevel, lr.sharedMeta[i]) + } + } + sort.Slice(filesInLevel, func(i, j int) bool { + return cmp(filesInLevel[i].Smallest.UserKey, filesInLevel[j].Smallest.UserKey) < 0 + }) + for i := 1; i < len(filesInLevel); i++ { + if sstableKeyCompare(cmp, filesInLevel[i-1].Largest, filesInLevel[i].Smallest) >= 0 { + return errors.AssertionFailedf("pebble: external shared sstables have overlapping ranges") + } } } return nil } -func ingestCleanup(objProvider *objstorage.Provider, meta []*fileMetadata) error { +func ingestCleanup(objProvider objstorage.Provider, meta []*fileMetadata) error { var firstErr error for i := range meta { - if err := objProvider.Remove(fileTypeTable, meta[i].FileNum); err != nil { + if err := objProvider.Remove(fileTypeTable, meta[i].FileBacking.DiskFileNum); err != nil { firstErr = firstError(firstErr, err) } } @@ -265,14 +540,21 @@ func ingestCleanup(objProvider *objstorage.Provider, meta []*fileMetadata) error } // ingestLink creates new objects which are backed by either hardlinks to or -// copies of the ingested files. +// copies of the ingested files. It also attaches shared objects to the provider. func ingestLink( - jobID int, opts *Options, objProvider *objstorage.Provider, paths []string, meta []*fileMetadata, + jobID int, + opts *Options, + objProvider objstorage.Provider, + lr ingestLoadResult, + shared []SharedSSTMeta, ) error { - for i := range paths { - objMeta, err := objProvider.LinkOrCopyFromLocal(opts.FS, paths[i], fileTypeTable, meta[i].FileNum) + for i := range lr.localPaths { + objMeta, err := objProvider.LinkOrCopyFromLocal( + context.TODO(), opts.FS, lr.localPaths[i], fileTypeTable, lr.localMeta[i].FileBacking.DiskFileNum, + objstorage.CreateOptions{PreferSharedStorage: true}, + ) if err != nil { - if err2 := ingestCleanup(objProvider, meta[:i]); err2 != nil { + if err2 := ingestCleanup(objProvider, lr.localMeta[:i]); err2 != nil { opts.Logger.Infof("ingest cleanup failed: %v", err2) } return err @@ -282,15 +564,57 @@ func ingestLink( JobID: jobID, Reason: "ingesting", Path: objProvider.Path(objMeta), - FileNum: meta[i].FileNum, + FileNum: lr.localMeta[i].FileNum, }) } } + sharedObjs := make([]objstorage.RemoteObjectToAttach, 0, len(shared)) + for i := range shared { + backing, err := shared[i].Backing.Get() + if err != nil { + return err + } + sharedObjs = append(sharedObjs, objstorage.RemoteObjectToAttach{ + FileNum: lr.sharedMeta[i].FileBacking.DiskFileNum, + FileType: fileTypeTable, + Backing: backing, + }) + } + sharedObjMetas, err := objProvider.AttachRemoteObjects(sharedObjs) + if err != nil { + return err + } + for i := range sharedObjMetas { + // One corner case around file sizes we need to be mindful of, is that + // if one of the shareObjs was initially created by us (and has boomeranged + // back from another node), we'll need to update the FileBacking's size + // to be the true underlying size. Otherwise, we could hit errors when we + // open the db again after a crash/restart (see checkConsistency in open.go), + // plus it more accurately allows us to prioritize compactions of files + // that were originally created by us. + if sharedObjMetas[i].IsShared() && !objProvider.IsSharedForeign(sharedObjMetas[i]) { + size, err := objProvider.Size(sharedObjMetas[i]) + if err != nil { + return err + } + lr.sharedMeta[i].FileBacking.Size = uint64(size) + } + if opts.EventListener.TableCreated != nil { + opts.EventListener.TableCreated(TableCreateInfo{ + JobID: jobID, + Reason: "ingesting", + Path: objProvider.Path(sharedObjMetas[i]), + FileNum: lr.sharedMeta[i].FileNum, + }) + } + } + // We do not need to do anything about lr.externalMetas. Those were already + // linked in ingestLoad. return nil } -func ingestMemtableOverlaps(cmp Compare, mem flushable, meta []*fileMetadata) bool { +func ingestMemtableOverlaps(cmp Compare, mem flushable, keyRanges []internalKeyRange) bool { iter := mem.newIter(nil) rangeDelIter := mem.newRangeDelIter(nil) rkeyIter := mem.newRangeKeyIter(nil) @@ -306,8 +630,8 @@ func ingestMemtableOverlaps(cmp Compare, mem flushable, meta []*fileMetadata) bo return err } - for _, m := range meta { - if overlapWithIterator(iter, &rangeDelIter, rkeyIter, m, cmp) { + for _, kr := range keyRanges { + if overlapWithIterator(iter, &rangeDelIter, rkeyIter, kr, cmp) { closeIters() return true } @@ -318,12 +642,12 @@ func ingestMemtableOverlaps(cmp Compare, mem flushable, meta []*fileMetadata) bo } func ingestUpdateSeqNum( - cmp Compare, format base.FormatKey, seqNum uint64, meta []*fileMetadata, + cmp Compare, format base.FormatKey, seqNum uint64, loadResult ingestLoadResult, ) error { setSeqFn := func(k base.InternalKey) base.InternalKey { return base.MakeInternalKey(k.UserKey, seqNum, k.Kind()) } - for _, m := range meta { + updateMetadata := func(m *fileMetadata) error { // NB: we set the fields directly here, rather than via their Extend* // methods, as we are updating sequence numbers. if m.HasPointKeys { @@ -355,25 +679,55 @@ func ingestUpdateSeqNum( return err } seqNum++ + return nil + } + + // Shared sstables are required to be sorted by level ascending. We then + // iterate the shared sstables in reverse, assigning the lower sequence + // numbers to the shared sstables that will be ingested into the lower + // (larger numbered) levels first. This ensures sequence number shadowing is + // correct. + for i := len(loadResult.sharedMeta) - 1; i >= 0; i-- { + if i-1 >= 0 && loadResult.sharedLevels[i-1] > loadResult.sharedLevels[i] { + panic(errors.AssertionFailedf("shared files %s, %s out of order", loadResult.sharedMeta[i-1], loadResult.sharedMeta[i])) + } + if err := updateMetadata(loadResult.sharedMeta[i]); err != nil { + return err + } + } + for i := range loadResult.localMeta { + if err := updateMetadata(loadResult.localMeta[i]); err != nil { + return err + } + } + for i := range loadResult.externalMeta { + if err := updateMetadata(loadResult.externalMeta[i]); err != nil { + return err + } } return nil } +// Denotes an internal key range. Smallest and largest are both inclusive. +type internalKeyRange struct { + smallest, largest InternalKey +} + func overlapWithIterator( iter internalIterator, rangeDelIter *keyspan.FragmentIterator, rkeyIter keyspan.FragmentIterator, - meta *fileMetadata, + keyRange internalKeyRange, cmp Compare, ) bool { // Check overlap with point operations. // // When using levelIter, it seeks to the SST whose boundaries - // contain meta.Smallest.UserKey(S). + // contain keyRange.smallest.UserKey(S). // It then tries to find a point in that SST that is >= S. // If there's no such point it means the SST ends in a tombstone in which case // levelIter.SeekGE generates a boundary range del sentinel. - // The comparison of this boundary with meta.Largest(L) below + // The comparison of this boundary with keyRange.largest(L) below // is subtle but maintains correctness. // 1) boundary < L, // since boundary is also > S (initial seek), @@ -385,9 +739,9 @@ func overlapWithIterator( // means boundary < L and hence is similar to 1). // 4) boundary == L and L is sentinel, // we'll always overlap since for any values of i,j ranges [i, k) and [j, k) always overlap. - key, _ := iter.SeekGE(meta.Smallest.UserKey, base.SeekGEFlagsNone) + key, _ := iter.SeekGE(keyRange.smallest.UserKey, base.SeekGEFlagsNone) if key != nil { - c := sstableKeyCompare(cmp, *key, meta.Largest) + c := sstableKeyCompare(cmp, *key, keyRange.largest) if c <= 0 { return true } @@ -399,7 +753,7 @@ func overlapWithIterator( computeOverlapWithSpans := func(rIter keyspan.FragmentIterator) bool { // NB: The spans surfaced by the fragment iterator are non-overlapping. - span := rIter.SeekLT(meta.Smallest.UserKey) + span := rIter.SeekLT(keyRange.smallest.UserKey) if span == nil { span = rIter.Next() } @@ -408,13 +762,13 @@ func overlapWithIterator( continue } key := span.SmallestKey() - c := sstableKeyCompare(cmp, key, meta.Largest) + c := sstableKeyCompare(cmp, key, keyRange.largest) if c > 0 { // The start of the span is after the largest key in the // ingested table. return false } - if cmp(span.End, meta.Smallest.UserKey) > 0 { + if cmp(span.End, keyRange.smallest.UserKey) > 0 { // The end of the span is greater than the smallest in the // table. Note that the span end key is exclusive, thus ">0" // instead of ">=0". @@ -443,16 +797,21 @@ func overlapWithIterator( return computeOverlapWithSpans(*rangeDelIter) } +// ingestTargetLevel returns the target level for a file being ingested. +// If suggestSplit is true, it accounts for ingest-time splitting as part of +// its target level calculation, and if a split candidate is found, that file +// is returned as the splitFile. func ingestTargetLevel( newIters tableNewIters, newRangeKeyIter keyspan.TableNewSpanIter, iterOps IterOptions, - cmp Compare, + comparer *Comparer, v *version, baseLevel int, compactions map[*compaction]struct{}, meta *fileMetadata, -) (int, error) { + suggestSplit bool, +) (targetLevel int, splitFile *fileMetadata, err error) { // Find the lowest level which does not have any files which overlap meta. We // search from L0 to L6 looking for whether there are any files in the level // which overlap meta. We want the "lowest" level (where lower means @@ -467,6 +826,14 @@ func ingestTargetLevel( // violate the sequence number invariant. // - no file boundary overlap with level i, since that will violate the // invariant that files do not overlap in levels i > 0. + // - if there is only a file overlap at a given level, and no data overlap, + // we can still slot a file at that level. We return the fileMetadata with + // which we have file boundary overlap (must be only one file, as sstable + // bounds are usually tight on user keys) and the caller is expected to split + // that sstable into two virtual sstables, allowing this file to go into that + // level. Note that if we have file boundary overlap with two files, which + // should only happen on rare occasions, we treat it as data overlap and + // don't use this optimization. // // The file boundary overlap check is simpler to conceptualize. Consider the // following example, in which the ingested file lies completely before or @@ -511,48 +878,46 @@ func ingestTargetLevel( // existing point that falls within the ingested table bounds as being "data // overlap". - targetLevel := 0 + // This assertion implicitly checks that we have the current version of + // the metadata. + if v.L0Sublevels == nil { + return 0, nil, errors.AssertionFailedf("could not read L0 sublevels") + } + // Check for overlap over the keys of L0 by iterating over the sublevels. + for subLevel := 0; subLevel < len(v.L0SublevelFiles); subLevel++ { + iter := newLevelIter(iterOps, comparer, newIters, + v.L0Sublevels.Levels[subLevel].Iter(), manifest.Level(0), internalIterOpts{}) - // Do we overlap with keys in L0? - // TODO(bananabrick): Use sublevels to compute overlap. - iter := v.Levels[0].Iter() - for meta0 := iter.First(); meta0 != nil; meta0 = iter.Next() { - c1 := sstableKeyCompare(cmp, meta.Smallest, meta0.Largest) - c2 := sstableKeyCompare(cmp, meta.Largest, meta0.Smallest) - if c1 > 0 || c2 < 0 { - continue - } + var rangeDelIter keyspan.FragmentIterator + // Pass in a non-nil pointer to rangeDelIter so that levelIter.findFileGE + // sets it up for the target file. + iter.initRangeDel(&rangeDelIter) - // TODO(sumeer): ingest is a user-facing operation, so we should accept a - // context and plumb it through, for tracing. - iter, rangeDelIter, err := newIters(context.Background(), meta0, nil, internalIterOpts{}) + levelIter := keyspan.LevelIter{} + levelIter.Init( + keyspan.SpanIterOptions{}, comparer.Compare, newRangeKeyIter, + v.L0Sublevels.Levels[subLevel].Iter(), manifest.Level(0), manifest.KeyTypeRange, + ) + + kr := internalKeyRange{ + smallest: meta.Smallest, + largest: meta.Largest, + } + overlap := overlapWithIterator(iter, &rangeDelIter, &levelIter, kr, comparer.Compare) + err := iter.Close() // Closes range del iter as well. + err = firstError(err, levelIter.Close()) if err != nil { - return 0, err - } - rkeyIter, err := newRangeKeyIter(meta0, nil) - if err != nil { - return 0, err - } - overlap := overlapWithIterator(iter, &rangeDelIter, rkeyIter, meta, cmp) - err = firstError(err, iter.Close()) - if rangeDelIter != nil { - err = firstError(err, rangeDelIter.Close()) - } - if rkeyIter != nil { - err = firstError(err, rkeyIter.Close()) - } - if err != nil { - return 0, err + return 0, nil, err } if overlap { - return targetLevel, nil + return targetLevel, nil, nil } } level := baseLevel for ; level < numLevels; level++ { - levelIter := newLevelIter(iterOps, cmp, nil /* split */, newIters, - v.Levels[level].Iter(), manifest.Level(level), nil) + levelIter := newLevelIter(iterOps, comparer, newIters, + v.Levels[level].Iter(), manifest.Level(level), internalIterOpts{}) var rangeDelIter keyspan.FragmentIterator // Pass in a non-nil pointer to rangeDelIter so that levelIter.findFileGE // sets it up for the target file. @@ -560,50 +925,76 @@ func ingestTargetLevel( rkeyLevelIter := &keyspan.LevelIter{} rkeyLevelIter.Init( - keyspan.SpanIterOptions{}, cmp, newRangeKeyIter, + keyspan.SpanIterOptions{}, comparer.Compare, newRangeKeyIter, v.Levels[level].Iter(), manifest.Level(level), manifest.KeyTypeRange, ) - overlap := overlapWithIterator(levelIter, &rangeDelIter, rkeyLevelIter, meta, cmp) + kr := internalKeyRange{ + smallest: meta.Smallest, + largest: meta.Largest, + } + overlap := overlapWithIterator(levelIter, &rangeDelIter, rkeyLevelIter, kr, comparer.Compare) err := levelIter.Close() // Closes range del iter as well. err = firstError(err, rkeyLevelIter.Close()) if err != nil { - return 0, err + return 0, nil, err } if overlap { - return targetLevel, nil + return targetLevel, splitFile, nil } // Check boundary overlap. - boundaryOverlaps := v.Overlaps(level, cmp, meta.Smallest.UserKey, + var candidateSplitFile *fileMetadata + boundaryOverlaps := v.Overlaps(level, comparer.Compare, meta.Smallest.UserKey, meta.Largest.UserKey, meta.Largest.IsExclusiveSentinel()) if !boundaryOverlaps.Empty() { - continue + // We are already guaranteed to not have any data overlaps with files + // in boundaryOverlaps, otherwise we'd have returned in the above if + // statements. Use this, plus boundaryOverlaps.Len() == 1 to detect for + // the case where we can slot this file into the current level despite + // a boundary overlap, by splitting one existing file into two virtual + // sstables. + if suggestSplit && boundaryOverlaps.Len() == 1 { + iter := boundaryOverlaps.Iter() + candidateSplitFile = iter.First() + } else { + // We either don't want to suggest ingest-time splits (i.e. + // !suggestSplit), or we boundary-overlapped with more than one file. + continue + } } - // Check boundary overlap with any ongoing compactions. + // Check boundary overlap with any ongoing compactions. We consider an + // overlapping compaction that's writing files to an output level as + // equivalent to boundary overlap with files in that output level. // - // We cannot check for data overlap with the new SSTs compaction will - // produce since compaction hasn't been done yet. However, there's no need - // to check since all keys in them will either be from c.startLevel or - // c.outputLevel, both levels having their data overlap already tested - // negative (else we'd have returned earlier). + // We cannot check for data overlap with the new SSTs compaction will produce + // since compaction hasn't been done yet. However, there's no need to check + // since all keys in them will be from levels in [c.startLevel, + // c.outputLevel], and all those levels have already had their data overlap + // tested negative (else we'd have returned earlier). + // + // An alternative approach would be to cancel these compactions and proceed + // with an ingest-time split on this level if necessary. However, compaction + // cancellation can result in significant wasted effort and is best avoided + // unless necessary. overlaps := false for c := range compactions { if c.outputLevel == nil || level != c.outputLevel.level { continue } - if cmp(meta.Smallest.UserKey, c.largest.UserKey) <= 0 && - cmp(meta.Largest.UserKey, c.smallest.UserKey) >= 0 { + if comparer.Compare(meta.Smallest.UserKey, c.largest.UserKey) <= 0 && + comparer.Compare(meta.Largest.UserKey, c.smallest.UserKey) >= 0 { overlaps = true break } } if !overlaps { targetLevel = level + splitFile = candidateSplitFile } } - return targetLevel, nil + return targetLevel, splitFile, nil } // Ingest ingests a set of sstables into the DB. Ingestion of the files is @@ -613,6 +1004,13 @@ func ingestTargetLevel( // the same filesystem as the DB. Sstables can be created for ingestion using // sstable.Writer. On success, Ingest removes the input paths. // +// Two types of sstables are accepted for ingestion(s): one is sstables present +// in the instance's vfs.FS and can be referenced locally. The other is sstables +// present in remote.Storage, referred to as shared or foreign sstables. These +// shared sstables can be linked through objstorageprovider.Provider, and do not +// need to already be present on the local vfs.FS. Foreign sstables must all fit +// in an excise span, and are destined for a level specified in SharedSSTMeta. +// // All sstables *must* be Sync()'d by the caller after all bytes are written // and before its file handle is closed; failure to do so could violate // durability or lead to corrupted on-disk state. This method cannot, in a @@ -624,21 +1022,28 @@ func ingestTargetLevel( // Ingestion loads each sstable into the lowest level of the LSM which it // doesn't overlap (see ingestTargetLevel). If an sstable overlaps a memtable, // ingestion forces the memtable to flush, and then waits for the flush to -// occur. +// occur. In some cases, such as with no foreign sstables and no excise span, +// ingestion that gets blocked on a memtable can join the flushable queue and +// finish even before the memtable has been flushed. // // The steps for ingestion are: // // 1. Allocate file numbers for every sstable being ingested. -// 2. Load the metadata for all sstables being ingest. -// 3. Sort the sstables by smallest key, verifying non overlap. -// 4. Hard link (or copy) the sstables into the DB directory. +// 2. Load the metadata for all sstables being ingested. +// 3. Sort the sstables by smallest key, verifying non overlap (for local +// sstables). +// 4. Hard link (or copy) the local sstables into the DB directory. // 5. Allocate a sequence number to use for all of the entries in the -// sstables. This is the step where overlap with memtables is +// local sstables. This is the step where overlap with memtables is // determined. If there is overlap, we remember the most recent memtable // that overlaps. -// 6. Update the sequence number in the ingested sstables. +// 6. Update the sequence number in the ingested local sstables. (Remote +// sstables get fixed sequence numbers that were determined at load time.) // 7. Wait for the most recent memtable that overlaps to flush (if any). // 8. Add the ingested sstables to the version (DB.ingestApply). +// 8.1. If an excise span was specified, figure out what sstables in the +// current version overlap with the excise span, and create new virtual +// sstables out of those sstables that exclude the excised span (DB.excise). // 9. Publish the ingestion sequence number. // // Note that if the mutable memtable overlaps with ingestion, a flush of the @@ -655,7 +1060,7 @@ func (d *DB) Ingest(paths []string) error { if d.opts.ReadOnly { return ErrReadOnly } - _, err := d.ingest(paths, ingestTargetLevel) + _, err := d.ingest(paths, ingestTargetLevel, nil /* shared */, KeyRange{}, nil /* external */) return err } @@ -675,6 +1080,32 @@ type IngestOperationStats struct { MemtableOverlappingFiles int } +// ExternalFile are external sstables that can be referenced through +// objprovider and ingested as remote files that will not be refcounted or +// cleaned up. For use with online restore. Note that the underlying sstable +// could contain keys outside the [Smallest,Largest) bounds; however Pebble +// is expected to only read the keys within those bounds. +type ExternalFile struct { + // Locator is the shared.Locator that can be used with objProvider to + // resolve a reference to this external sstable. + Locator remote.Locator + // ObjName is the unique name of this sstable on Locator. + ObjName string + // Size of the referenced proportion of the virtualized sstable. An estimate + // is acceptable in lieu of the backing file size. + Size uint64 + // SmallestUserKey and LargestUserKey are the [smallest,largest) user key + // bounds of the sstable. Both these bounds are loose i.e. it's possible for + // the sstable to not span the entirety of this range. However, multiple + // ExternalFiles in one ingestion must all have non-overlapping + // [smallest, largest) spans. Note that this Largest bound is exclusive. + SmallestUserKey, LargestUserKey []byte + // HasPointKey and HasRangeKey denote whether this file contains point keys + // or range keys. If both structs are false, an error is returned during + // ingestion. + HasPointKey, HasRangeKey bool +} + // IngestWithStats does the same as Ingest, and additionally returns // IngestOperationStats. func (d *DB) IngestWithStats(paths []string) (IngestOperationStats, error) { @@ -684,7 +1115,47 @@ func (d *DB) IngestWithStats(paths []string) (IngestOperationStats, error) { if d.opts.ReadOnly { return IngestOperationStats{}, ErrReadOnly } - return d.ingest(paths, ingestTargetLevel) + return d.ingest(paths, ingestTargetLevel, nil /* shared */, KeyRange{}, nil /* external */) +} + +// IngestExternalFiles does the same as IngestWithStats, and additionally +// accepts external files (with locator info that can be resolved using +// d.opts.SharedStorage). These files must also be non-overlapping with +// each other, and must be resolvable through d.objProvider. +func (d *DB) IngestExternalFiles(external []ExternalFile) (IngestOperationStats, error) { + if err := d.closed.Load(); err != nil { + panic(err) + } + + if d.opts.ReadOnly { + return IngestOperationStats{}, ErrReadOnly + } + if d.opts.Experimental.RemoteStorage == nil { + return IngestOperationStats{}, errors.New("pebble: cannot ingest external files without shared storage configured") + } + return d.ingest(nil, ingestTargetLevel, nil /* shared */, KeyRange{}, external) +} + +// IngestAndExcise does the same as IngestWithStats, and additionally accepts a +// list of shared files to ingest that can be read from a remote.Storage through +// a Provider. All the shared files must live within exciseSpan, and any existing +// keys in exciseSpan are deleted by turning existing sstables into virtual +// sstables (if not virtual already) and shrinking their spans to exclude +// exciseSpan. See the comment at Ingest for a more complete picture of the +// ingestion process. +// +// Panics if this DB instance was not instantiated with a remote.Storage and +// shared sstables are present. +func (d *DB) IngestAndExcise( + paths []string, shared []SharedSSTMeta, exciseSpan KeyRange, +) (IngestOperationStats, error) { + if err := d.closed.Load(); err != nil { + panic(err) + } + if d.opts.ReadOnly { + return IngestOperationStats{}, ErrReadOnly + } + return d.ingest(paths, ingestTargetLevel, shared, exciseSpan, nil /* external */) } // Both DB.mu and commitPipeline.mu must be held while this is called. @@ -700,12 +1171,12 @@ func (d *DB) newIngestedFlushableEntry( // time, then we'll lose the ingest sequence number information. But this // information will also be reconstructed on node restart. if err := ingestUpdateSeqNum( - d.cmp, d.opts.Comparer.FormatKey, seqNum, meta, + d.cmp, d.opts.Comparer.FormatKey, seqNum, ingestLoadResult{localMeta: meta}, ); err != nil { return nil, err } - f := newIngestedFlushable(meta, d.cmp, d.split, d.newIters, d.tableNewRangeKeyIter) + f := newIngestedFlushable(meta, d.opts.Comparer, d.newIters, d.tableNewRangeKeyIter) // NB: The logNum/seqNum are the WAL number which we're writing this entry // to and the sequence number within the WAL which we'll write this entry @@ -714,13 +1185,13 @@ func (d *DB) newIngestedFlushableEntry( // The flushable entry starts off with a single reader ref, so increment // the FileMetadata.Refs. for _, file := range f.files { - atomic.AddInt32(&file.Refs, 1) + file.Ref() } - entry.unrefFiles = func() []*fileMetadata { - var obsolete []*fileMetadata + entry.unrefFiles = func() []*fileBacking { + var obsolete []*fileBacking for _, file := range f.files { - if val := atomic.AddInt32(&file.Refs, -1); val == 0 { - obsolete = append(obsolete, file) + if file.Unref() == 0 { + obsolete = append(obsolete, file.FileMetadata.FileBacking) } } return obsolete @@ -794,36 +1265,49 @@ func (d *DB) handleIngestAsFlushable(meta []*fileMetadata, seqNum uint64) error return nil } +// See comment at Ingest() for details on how this works. func (d *DB) ingest( - paths []string, targetLevelFunc ingestTargetLevelFunc, + paths []string, + targetLevelFunc ingestTargetLevelFunc, + shared []SharedSSTMeta, + exciseSpan KeyRange, + external []ExternalFile, ) (IngestOperationStats, error) { + if len(shared) > 0 && d.opts.Experimental.RemoteStorage == nil { + panic("cannot ingest shared sstables with nil SharedStorage") + } + if (exciseSpan.Valid() || len(shared) > 0 || len(external) > 0) && d.FormatMajorVersion() < FormatVirtualSSTables { + return IngestOperationStats{}, errors.New("pebble: format major version too old for excise, shared or external sstable ingestion") + } // Allocate file numbers for all of the files being ingested and mark them as // pending in order to prevent them from being deleted. Note that this causes // the file number ordering to be out of alignment with sequence number // ordering. The sorting of L0 tables by sequence number avoids relying on // that (busted) invariant. d.mu.Lock() - pendingOutputs := make([]FileNum, len(paths)) - for i := range paths { - pendingOutputs[i] = d.mu.versions.getNextFileNum() + pendingOutputs := make([]base.DiskFileNum, len(paths)+len(shared)+len(external)) + for i := 0; i < len(paths)+len(shared)+len(external); i++ { + pendingOutputs[i] = d.mu.versions.getNextFileNum().DiskFileNum() } + jobID := d.mu.nextJobID d.mu.nextJobID++ d.mu.Unlock() - // Load the metadata for all of the files being ingested. This step detects + // Load the metadata for all the files being ingested. This step detects // and elides empty sstables. - meta, paths, err := ingestLoad(d.opts, d.FormatMajorVersion(), paths, d.cacheID, pendingOutputs) + loadResult, err := ingestLoad(d.opts, d.FormatMajorVersion(), paths, shared, external, d.cacheID, pendingOutputs, d.objProvider, jobID) if err != nil { return IngestOperationStats{}, err } - if len(meta) == 0 { + + if loadResult.fileCount == 0 { // All of the sstables to be ingested were empty. Nothing to do. return IngestOperationStats{}, nil } // Verify the sstables do not overlap. - if err := ingestSortAndVerify(d.cmp, meta, paths); err != nil { + if err := ingestSortAndVerify(d.cmp, loadResult, exciseSpan); err != nil { return IngestOperationStats{}, err } @@ -832,9 +1316,10 @@ func (d *DB) ingest( // (e.g. because the files reside on a different filesystem), ingestLink will // fall back to copying, and if that fails we undo our work and return an // error. - if err := ingestLink(jobID, d.opts, d.objProvider, paths, meta); err != nil { + if err := ingestLink(jobID, d.opts, d.objProvider, loadResult, shared); err != nil { return IngestOperationStats{}, err } + // Make the new tables durable. We need to do this at some point before we // update the MANIFEST (via logAndApply), otherwise a crash can have the // tables referenced in the MANIFEST, but not present in the provider. @@ -845,7 +1330,7 @@ func (d *DB) ingest( // metaFlushableOverlaps is a slice parallel to meta indicating which of the // ingested sstables overlap some table in the flushable queue. It's used to // approximate ingest-into-L0 stats when using flushable ingests. - metaFlushableOverlaps := make([]bool, len(meta)) + metaFlushableOverlaps := make([]bool, loadResult.fileCount) var mem *flushableEntry var mut *memTable // asFlushable indicates whether the sstable was ingested as a flushable. @@ -866,12 +1351,17 @@ func (d *DB) ingest( iter := m.newIter(nil) rangeDelIter := m.newRangeDelIter(nil) rkeyIter := m.newRangeKeyIter(nil) - for i := range meta { + + checkForOverlap := func(i int, meta *fileMetadata) { if metaFlushableOverlaps[i] { // This table already overlapped a more recent flushable. - continue + return } - if overlapWithIterator(iter, &rangeDelIter, rkeyIter, meta[i], d.cmp) { + kr := internalKeyRange{ + smallest: meta.Smallest, + largest: meta.Largest, + } + if overlapWithIterator(iter, &rangeDelIter, rkeyIter, kr, d.cmp) { // If this is the first table to overlap a flushable, save // the flushable. This ingest must be ingested or flushed // after it. @@ -881,6 +1371,26 @@ func (d *DB) ingest( metaFlushableOverlaps[i] = true } } + for i := range loadResult.localMeta { + checkForOverlap(i, loadResult.localMeta[i]) + } + for i := range loadResult.sharedMeta { + checkForOverlap(len(loadResult.localMeta)+i, loadResult.sharedMeta[i]) + } + for i := range loadResult.externalMeta { + checkForOverlap(len(loadResult.localMeta)+len(loadResult.sharedMeta)+i, loadResult.externalMeta[i]) + } + if exciseSpan.Valid() { + kr := internalKeyRange{ + smallest: base.MakeInternalKey(exciseSpan.Start, InternalKeySeqNumMax, InternalKeyKindMax), + largest: base.MakeExclusiveSentinelKey(InternalKeyKindRangeDelete, exciseSpan.End), + } + if overlapWithIterator(iter, &rangeDelIter, rkeyIter, kr, d.cmp) { + if mem == nil { + mem = m + } + } + } err := iter.Close() if rangeDelIter != nil { err = firstError(err, rangeDelIter.Close()) @@ -911,9 +1421,16 @@ func (d *DB) ingest( // The ingestion overlaps with some entry in the flushable queue. if d.FormatMajorVersion() < FormatFlushableIngest || d.opts.Experimental.DisableIngestAsFlushable() || + len(shared) > 0 || exciseSpan.Valid() || len(external) > 0 || (len(d.mu.mem.queue) > d.opts.MemTableStopWritesThreshold-1) { // We're not able to ingest as a flushable, // so we must synchronously flush. + // + // TODO(bilal): Currently, if any of the files being ingested are shared or + // there's an excise span present, we cannot use flushable ingests and need + // to wait synchronously. Either remove this caveat by fleshing out + // flushable ingest logic to also account for these cases, or remove this + // comment. Tracking issue: https://github.com/cockroachdb/pebble/issues/2676 if mem.flushable == d.mu.mem.mutable { err = d.makeRoomForWrite(nil) } @@ -933,7 +1450,7 @@ func (d *DB) ingest( // Since there aren't too many memtables already queued up, we can // slide the ingested sstables on top of the existing memtables. asFlushable = true - err = d.handleIngestAsFlushable(meta, seqNum) + err = d.handleIngestAsFlushable(loadResult.localMeta, seqNum) } var ve *versionEdit @@ -950,12 +1467,12 @@ func (d *DB) ingest( return } - // Update the sequence number for all of the sstables in the - // metadata. Writing the metadata to the manifest when the - // version edit is applied is the mechanism that persists the - // sequence number. The sstables themselves are left unmodified. + // Update the sequence numbers for all ingested sstables' + // metadata. When the version edit is applied, the metadata is + // written to the manifest, persisting the sequence number. + // The sstables themselves are left unmodified. if err = ingestUpdateSeqNum( - d.cmp, d.opts.Comparer.FormatKey, seqNum, meta, + d.cmp, d.opts.Comparer.FormatKey, seqNum, loadResult, ); err != nil { if mut != nil { if mut.writerUnref() { @@ -975,30 +1492,49 @@ func (d *DB) ingest( // Assign the sstables to the correct level in the LSM and apply the // version edit. - ve, err = d.ingestApply(jobID, meta, targetLevelFunc, mut) + ve, err = d.ingestApply(jobID, loadResult, targetLevelFunc, mut, exciseSpan) } - d.commit.AllocateSeqNum(len(meta), prepare, apply) + // Only one ingest can occur at a time because if not, one would block waiting + // for the other to finish applying. This blocking would happen while holding + // the commit mutex which would prevent unrelated batches from writing their + // changes to the WAL and memtable. This will cause a bigger commit hiccup + // during ingestion. + d.commit.ingestSem <- struct{}{} + d.commit.AllocateSeqNum(loadResult.fileCount, prepare, apply) + <-d.commit.ingestSem if err != nil { - if err2 := ingestCleanup(d.objProvider, meta); err2 != nil { + if err2 := ingestCleanup(d.objProvider, loadResult.localMeta); err2 != nil { d.opts.Logger.Infof("ingest cleanup failed: %v", err2) } } else { // Since we either created a hard link to the ingesting files, or copied // them over, it is safe to remove the originals paths. - for _, path := range paths { + for _, path := range loadResult.localPaths { if err2 := d.opts.FS.Remove(path); err2 != nil { d.opts.Logger.Infof("ingest failed to remove original file: %s", err2) } } } + if invariants.Enabled { + for _, sharedMeta := range loadResult.sharedMeta { + d.checkVirtualBounds(sharedMeta) + } + } + info := TableIngestInfo{ - JobID: jobID, - GlobalSeqNum: meta[0].SmallestSeqNum, - Err: err, - flushable: asFlushable, + JobID: jobID, + Err: err, + flushable: asFlushable, + } + if len(loadResult.localMeta) > 0 { + info.GlobalSeqNum = loadResult.localMeta[0].SmallestSeqNum + } else if len(loadResult.sharedMeta) > 0 { + info.GlobalSeqNum = loadResult.sharedMeta[0].SmallestSeqNum + } else { + info.GlobalSeqNum = loadResult.externalMeta[0].SmallestSeqNum } var stats IngestOperationStats if ve != nil { @@ -1014,16 +1550,17 @@ func (d *DB) ingest( if e.Level == 0 { stats.ApproxIngestedIntoL0Bytes += e.Meta.Size } - if metaFlushableOverlaps[i] { + if i < len(metaFlushableOverlaps) && metaFlushableOverlaps[i] { stats.MemtableOverlappingFiles++ } } } else if asFlushable { + // NB: If asFlushable == true, there are no shared sstables. info.Tables = make([]struct { TableInfo Level int - }, len(meta)) - for i, f := range meta { + }, len(loadResult.localMeta)) + for i, f := range loadResult.localMeta { info.Tables[i].Level = -1 info.Tables[i].TableInfo = f.TableInfo() stats.Bytes += f.Size @@ -1046,25 +1583,422 @@ func (d *DB) ingest( return stats, err } +// excise updates ve to include a replacement of the file m with new virtual +// sstables that exclude exciseSpan, returning a slice of newly-created files if +// any. If the entirety of m is deleted by exciseSpan, no new sstables are added +// and m is deleted. Note that ve is updated in-place. +// +// The manifest lock must be held when calling this method. +func (d *DB) excise( + exciseSpan KeyRange, m *fileMetadata, ve *versionEdit, level int, +) ([]manifest.NewFileEntry, error) { + numCreatedFiles := 0 + // Check if there's actually an overlap between m and exciseSpan. + if !exciseSpan.Overlaps(d.cmp, m) { + return nil, nil + } + ve.DeletedFiles[deletedFileEntry{ + Level: level, + FileNum: m.FileNum, + }] = m + // Fast path: m sits entirely within the exciseSpan, so just delete it. + if exciseSpan.Contains(d.cmp, m.Smallest) && exciseSpan.Contains(d.cmp, m.Largest) { + return nil, nil + } + var iter internalIterator + var rangeDelIter keyspan.FragmentIterator + var rangeKeyIter keyspan.FragmentIterator + needsBacking := false + // Create a file to the left of the excise span, if necessary. + // The bounds of this file will be [m.Smallest, lastKeyBefore(exciseSpan.Start)]. + // + // We create bounds that are tight on user keys, and we make the effort to find + // the last key in the original sstable that's smaller than exciseSpan.Start + // even though it requires some sstable reads. We could choose to create + // virtual sstables on loose userKey bounds, in which case we could just set + // leftFile.Largest to an exclusive sentinel at exciseSpan.Start. The biggest + // issue with that approach would be that it'd lead to lots of small virtual + // sstables in the LSM that have no guarantee on containing even a single user + // key within the file bounds. This has the potential to increase both read and + // write-amp as we will be opening up these sstables only to find no relevant + // keys in the read path, and compacting sstables on top of them instead of + // directly into the space occupied by them. We choose to incur the cost of + // calculating tight bounds at this time instead of creating more work in the + // future. + // + // TODO(bilal): Some of this work can happen without grabbing the manifest + // lock; we could grab one currentVersion, release the lock, calculate excised + // files, then grab the lock again and recalculate for just the files that + // have changed since our previous calculation. Do this optimiaztino as part of + // https://github.com/cockroachdb/pebble/issues/2112 . + if d.cmp(m.Smallest.UserKey, exciseSpan.Start) < 0 { + leftFile := &fileMetadata{ + Virtual: true, + FileBacking: m.FileBacking, + FileNum: d.mu.versions.getNextFileNum(), + // Note that these are loose bounds for smallest/largest seqnums, but they're + // sufficient for maintaining correctness. + SmallestSeqNum: m.SmallestSeqNum, + LargestSeqNum: m.LargestSeqNum, + } + if m.HasPointKeys && !exciseSpan.Contains(d.cmp, m.SmallestPointKey) { + // This file will contain point keys + smallestPointKey := m.SmallestPointKey + var err error + iter, rangeDelIter, err = d.newIters(context.TODO(), m, &IterOptions{level: manifest.Level(level)}, internalIterOpts{}) + if err != nil { + return nil, err + } + var key *InternalKey + if iter != nil { + defer iter.Close() + key, _ = iter.SeekLT(exciseSpan.Start, base.SeekLTFlagsNone) + } else { + iter = emptyIter + } + if key != nil { + leftFile.ExtendPointKeyBounds(d.cmp, smallestPointKey, key.Clone()) + } + // Store the min of (exciseSpan.Start, rdel.End) in lastRangeDel. This + // needs to be a copy if the key is owned by the range del iter. + var lastRangeDel []byte + if rangeDelIter != nil { + defer rangeDelIter.Close() + rdel := rangeDelIter.SeekLT(exciseSpan.Start) + if rdel != nil { + lastRangeDel = append(lastRangeDel[:0], rdel.End...) + if d.cmp(lastRangeDel, exciseSpan.Start) > 0 { + lastRangeDel = exciseSpan.Start + } + } + } else { + rangeDelIter = emptyKeyspanIter + } + if lastRangeDel != nil { + leftFile.ExtendPointKeyBounds(d.cmp, smallestPointKey, base.MakeExclusiveSentinelKey(InternalKeyKindRangeDelete, lastRangeDel)) + } + } + if m.HasRangeKeys && !exciseSpan.Contains(d.cmp, m.SmallestRangeKey) { + // This file will contain range keys + var err error + smallestRangeKey := m.SmallestRangeKey + rangeKeyIter, err = d.tableNewRangeKeyIter(m, keyspan.SpanIterOptions{}) + if err != nil { + return nil, err + } + // Store the min of (exciseSpan.Start, rkey.End) in lastRangeKey. This + // needs to be a copy if the key is owned by the range key iter. + var lastRangeKey []byte + var lastRangeKeyKind InternalKeyKind + defer rangeKeyIter.Close() + rkey := rangeKeyIter.SeekLT(exciseSpan.Start) + if rkey != nil { + lastRangeKey = append(lastRangeKey[:0], rkey.End...) + if d.cmp(lastRangeKey, exciseSpan.Start) > 0 { + lastRangeKey = exciseSpan.Start + } + lastRangeKeyKind = rkey.Keys[0].Kind() + } + if lastRangeKey != nil { + leftFile.ExtendRangeKeyBounds(d.cmp, smallestRangeKey, base.MakeExclusiveSentinelKey(lastRangeKeyKind, lastRangeKey)) + } + } + if leftFile.HasRangeKeys || leftFile.HasPointKeys { + var err error + leftFile.Size, err = d.tableCache.estimateSize(m, leftFile.Smallest.UserKey, leftFile.Largest.UserKey) + if err != nil { + return nil, err + } + if leftFile.Size == 0 { + // On occasion, estimateSize gives us a low estimate, i.e. a 0 file size, + // such as if the excised file only has range keys/dels and no point + // keys. This can cause panics in places where we divide by file sizes. + // Correct for it here. + leftFile.Size = 1 + } + if err := leftFile.Validate(d.cmp, d.opts.Comparer.FormatKey); err != nil { + return nil, err + } + leftFile.ValidateVirtual(m) + d.checkVirtualBounds(leftFile) + ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: level, Meta: leftFile}) + needsBacking = true + numCreatedFiles++ + } + } + // Create a file to the right, if necessary. + if exciseSpan.Contains(d.cmp, m.Largest) { + // No key exists to the right of the excise span in this file. + if needsBacking && !m.Virtual { + // If m is virtual, then its file backing is already known to the manifest. + // We don't need to create another file backing. Note that there must be + // only one CreatedBackingTables entry per backing sstable. This is + // indicated by the VersionEdit.CreatedBackingTables invariant. + ve.CreatedBackingTables = append(ve.CreatedBackingTables, m.FileBacking) + } + return ve.NewFiles[len(ve.NewFiles)-numCreatedFiles:], nil + } + // Create a new file, rightFile, between [firstKeyAfter(exciseSpan.End), m.Largest]. + // + // See comment before the definition of leftFile for the motivation behind + // calculating tight user-key bounds. + rightFile := &fileMetadata{ + Virtual: true, + FileBacking: m.FileBacking, + FileNum: d.mu.versions.getNextFileNum(), + // Note that these are loose bounds for smallest/largest seqnums, but they're + // sufficient for maintaining correctness. + SmallestSeqNum: m.SmallestSeqNum, + LargestSeqNum: m.LargestSeqNum, + } + if m.HasPointKeys && !exciseSpan.Contains(d.cmp, m.LargestPointKey) { + // This file will contain point keys + largestPointKey := m.LargestPointKey + var err error + if iter == nil && rangeDelIter == nil { + iter, rangeDelIter, err = d.newIters(context.TODO(), m, &IterOptions{level: manifest.Level(level)}, internalIterOpts{}) + if err != nil { + return nil, err + } + if iter != nil { + defer iter.Close() + } else { + iter = emptyIter + } + if rangeDelIter != nil { + defer rangeDelIter.Close() + } else { + rangeDelIter = emptyKeyspanIter + } + } + key, _ := iter.SeekGE(exciseSpan.End, base.SeekGEFlagsNone) + if key != nil { + rightFile.ExtendPointKeyBounds(d.cmp, key.Clone(), largestPointKey) + } + // Store the max of (exciseSpan.End, rdel.Start) in firstRangeDel. This + // needs to be a copy if the key is owned by the range del iter. + var firstRangeDel []byte + rdel := rangeDelIter.SeekGE(exciseSpan.End) + if rdel != nil { + firstRangeDel = append(firstRangeDel[:0], rdel.Start...) + if d.cmp(firstRangeDel, exciseSpan.End) < 0 { + firstRangeDel = exciseSpan.End + } + } + if firstRangeDel != nil { + smallestPointKey := rdel.SmallestKey() + smallestPointKey.UserKey = firstRangeDel + rightFile.ExtendPointKeyBounds(d.cmp, smallestPointKey, largestPointKey) + } + } + if m.HasRangeKeys && !exciseSpan.Contains(d.cmp, m.LargestRangeKey) { + // This file will contain range keys. + largestRangeKey := m.LargestRangeKey + if rangeKeyIter == nil { + var err error + rangeKeyIter, err = d.tableNewRangeKeyIter(m, keyspan.SpanIterOptions{}) + if err != nil { + return nil, err + } + defer rangeKeyIter.Close() + } + // Store the max of (exciseSpan.End, rkey.Start) in firstRangeKey. This + // needs to be a copy if the key is owned by the range key iter. + var firstRangeKey []byte + rkey := rangeKeyIter.SeekGE(exciseSpan.End) + if rkey != nil { + firstRangeKey = append(firstRangeKey[:0], rkey.Start...) + if d.cmp(firstRangeKey, exciseSpan.End) < 0 { + firstRangeKey = exciseSpan.End + } + } + if firstRangeKey != nil { + smallestRangeKey := rkey.SmallestKey() + smallestRangeKey.UserKey = firstRangeKey + // We call ExtendRangeKeyBounds so any internal boundType fields are + // set correctly. Note that this is mildly wasteful as we'll be comparing + // rightFile.{Smallest,Largest}RangeKey with themselves, which can be + // avoided if we exported ExtendOverallKeyBounds or so. + rightFile.ExtendRangeKeyBounds(d.cmp, smallestRangeKey, largestRangeKey) + } + } + if rightFile.HasRangeKeys || rightFile.HasPointKeys { + var err error + rightFile.Size, err = d.tableCache.estimateSize(m, rightFile.Smallest.UserKey, rightFile.Largest.UserKey) + if err != nil { + return nil, err + } + if rightFile.Size == 0 { + // On occasion, estimateSize gives us a low estimate, i.e. a 0 file size, + // such as if the excised file only has range keys/dels and no point keys. + // This can cause panics in places where we divide by file sizes. Correct + // for it here. + rightFile.Size = 1 + } + rightFile.ValidateVirtual(m) + d.checkVirtualBounds(rightFile) + ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: level, Meta: rightFile}) + needsBacking = true + numCreatedFiles++ + } + + if needsBacking && !m.Virtual { + // If m is virtual, then its file backing is already known to the manifest. + // We don't need to create another file backing. Note that there must be + // only one CreatedBackingTables entry per backing sstable. This is + // indicated by the VersionEdit.CreatedBackingTables invariant. + ve.CreatedBackingTables = append(ve.CreatedBackingTables, m.FileBacking) + } + + if err := rightFile.Validate(d.cmp, d.opts.Comparer.FormatKey); err != nil { + return nil, err + } + return ve.NewFiles[len(ve.NewFiles)-numCreatedFiles:], nil +} + type ingestTargetLevelFunc func( newIters tableNewIters, newRangeKeyIter keyspan.TableNewSpanIter, iterOps IterOptions, - cmp Compare, + comparer *Comparer, v *version, baseLevel int, compactions map[*compaction]struct{}, meta *fileMetadata, -) (int, error) + suggestSplit bool, +) (int, *fileMetadata, error) + +type ingestSplitFile struct { + // ingestFile is the file being ingested. + ingestFile *fileMetadata + // splitFile is the file that needs to be split to allow ingestFile to slot + // into `level` level. + splitFile *fileMetadata + // The level where ingestFile will go (and where splitFile already is). + level int +} + +// ingestSplit splits files specified in `files` and updates ve in-place to +// account for existing files getting split into two virtual sstables. The map +// `replacedFiles` contains an in-progress map of all files that have been +// replaced with new virtual sstables in this version edit so far, which is also +// updated in-place. +// +// d.mu as well as the manifest lock must be held when calling this method. +func (d *DB) ingestSplit( + ve *versionEdit, + updateMetrics func(*fileMetadata, int, []newFileEntry), + files []ingestSplitFile, + replacedFiles map[base.FileNum][]newFileEntry, +) error { + for _, s := range files { + // replacedFiles can be thought of as a tree, where we start iterating with + // s.splitFile and run its fileNum through replacedFiles, then find which of + // the replaced files overlaps with s.ingestFile, which becomes the new + // splitFile, then we check splitFile's replacements in replacedFiles again + // for overlap with s.ingestFile, and so on until we either can't find the + // current splitFile in replacedFiles (i.e. that's the file that now needs to + // be split), or we don't find a file that overlaps with s.ingestFile, which + // means a prior ingest split already produced enough room for s.ingestFile + // to go into this level without necessitating another ingest split. + splitFile := s.splitFile + for splitFile != nil { + replaced, ok := replacedFiles[splitFile.FileNum] + if !ok { + break + } + updatedSplitFile := false + for i := range replaced { + if replaced[i].Meta.Overlaps(d.cmp, s.ingestFile.Smallest.UserKey, s.ingestFile.Largest.UserKey, s.ingestFile.Largest.IsExclusiveSentinel()) { + if updatedSplitFile { + // This should never happen because the earlier ingestTargetLevel + // function only finds split file candidates that are guaranteed to + // have no data overlap, only boundary overlap. See the comments + // in that method to see the definitions of data vs boundary + // overlap. That, plus the fact that files in `replaced` are + // guaranteed to have file bounds that are tight on user keys + // (as that's what `d.excise` produces), means that the only case + // where we overlap with two or more files in `replaced` is if we + // actually had data overlap all along, or if the ingestion files + // were overlapping, either of which is an invariant violation. + panic("updated with two files in ingestSplit") + } + splitFile = replaced[i].Meta + updatedSplitFile = true + } + } + if !updatedSplitFile { + // None of the replaced files overlapped with the file being ingested. + // This can happen if we've already excised a span overlapping with + // this file, or if we have consecutive ingested files that can slide + // within the same gap between keys in an existing file. For instance, + // if an existing file has keys a and g and we're ingesting b-c, d-e, + // the first loop iteration will split the existing file into one that + // ends in a and another that starts at g, and the second iteration will + // fall into this case and require no splitting. + // + // No splitting necessary. + splitFile = nil + } + } + if splitFile == nil { + continue + } + // NB: excise operates on [start, end). We're splitting at [start, end] + // (assuming !s.ingestFile.Largest.IsExclusiveSentinel()). The conflation + // of exclusive vs inclusive end bounds should not make a difference here + // as we're guaranteed to not have any data overlap between splitFile and + // s.ingestFile, so panic if we do see a newly added file with an endKey + // equalling s.ingestFile.Largest, and !s.ingestFile.Largest.IsExclusiveSentinel() + added, err := d.excise(KeyRange{Start: s.ingestFile.Smallest.UserKey, End: s.ingestFile.Largest.UserKey}, splitFile, ve, s.level) + if err != nil { + return err + } + if _, ok := ve.DeletedFiles[deletedFileEntry{ + Level: s.level, + FileNum: splitFile.FileNum, + }]; !ok { + panic("did not split file that was expected to be split") + } + replacedFiles[splitFile.FileNum] = added + for i := range added { + if s.ingestFile.Overlaps(d.cmp, added[i].Meta.Smallest.UserKey, added[i].Meta.Largest.UserKey, added[i].Meta.Largest.IsExclusiveSentinel()) { + panic("ingest-time split produced a file that overlaps with ingested file") + } + } + updateMetrics(splitFile, s.level, added) + } + // Flatten the version edit by removing any entries from ve.NewFiles that + // are also in ve.DeletedFiles. + newNewFiles := ve.NewFiles[:0] + for i := range ve.NewFiles { + fn := ve.NewFiles[i].Meta.FileNum + deEntry := deletedFileEntry{Level: ve.NewFiles[i].Level, FileNum: fn} + if _, ok := ve.DeletedFiles[deEntry]; ok { + delete(ve.DeletedFiles, deEntry) + } else { + newNewFiles = append(newNewFiles, ve.NewFiles[i]) + } + } + ve.NewFiles = newNewFiles + return nil +} func (d *DB) ingestApply( - jobID int, meta []*fileMetadata, findTargetLevel ingestTargetLevelFunc, mut *memTable, + jobID int, + lr ingestLoadResult, + findTargetLevel ingestTargetLevelFunc, + mut *memTable, + exciseSpan KeyRange, ) (*versionEdit, error) { d.mu.Lock() defer d.mu.Unlock() ve := &versionEdit{ - NewFiles: make([]newFileEntry, len(meta)), + NewFiles: make([]newFileEntry, lr.fileCount), + } + if exciseSpan.Valid() || (d.opts.Experimental.IngestSplit != nil && d.opts.Experimental.IngestSplit()) { + ve.DeletedFiles = map[manifest.DeletedFileEntry]*manifest.FileMetadata{} } metrics := make(map[int]*LevelMetrics) @@ -1076,26 +2010,101 @@ func (d *DB) ingestApply( // returns must unlock the manifest. d.mu.versions.logLock() - scheduleFlush := false if mut != nil { // Unref the mutable memtable to allows its flush to proceed. Now that we've // acquired the manifest lock, we can be certain that if the mutable // memtable has received more recent conflicting writes, the flush won't // beat us to applying to the manifest resulting in sequence number - // inversion. - scheduleFlush = mut.writerUnref() + // inversion. Even though we call maybeScheduleFlush right now, this flush + // will apply after our ingestion. + if mut.writerUnref() { + d.maybeScheduleFlush() + } } + shouldIngestSplit := d.opts.Experimental.IngestSplit != nil && + d.opts.Experimental.IngestSplit() && d.FormatMajorVersion() >= FormatVirtualSSTables current := d.mu.versions.currentVersion() baseLevel := d.mu.versions.picker.getBaseLevel() iterOps := IterOptions{logger: d.opts.Logger} - for i := range meta { + // filesToSplit is a list where each element is a pair consisting of a file + // being ingested and a file being split to make room for an ingestion into + // that level. Each ingested file will appear at most once in this list. It + // is possible for split files to appear twice in this list. + filesToSplit := make([]ingestSplitFile, 0) + checkCompactions := false + for i := 0; i < lr.fileCount; i++ { // Determine the lowest level in the LSM for which the sstable doesn't // overlap any existing files in the level. - m := meta[i] + var m *fileMetadata + sharedIdx := -1 + sharedLevel := -1 + externalFile := false + if i < len(lr.localMeta) { + // local file. + m = lr.localMeta[i] + } else if (i - len(lr.localMeta)) < len(lr.sharedMeta) { + // shared file. + sharedIdx = i - len(lr.localMeta) + m = lr.sharedMeta[sharedIdx] + sharedLevel = int(lr.sharedLevels[sharedIdx]) + } else { + // external file. + externalFile = true + m = lr.externalMeta[i-(len(lr.localMeta)+len(lr.sharedMeta))] + } f := &ve.NewFiles[i] var err error - f.Level, err = findTargetLevel(d.newIters, d.tableNewRangeKeyIter, iterOps, d.cmp, current, baseLevel, d.mu.compact.inProgress, m) + if sharedIdx >= 0 { + f.Level = sharedLevel + if f.Level < sharedLevelsStart { + panic("cannot slot a shared file higher than the highest shared level") + } + ve.CreatedBackingTables = append(ve.CreatedBackingTables, m.FileBacking) + } else { + if externalFile { + ve.CreatedBackingTables = append(ve.CreatedBackingTables, m.FileBacking) + } + var splitFile *fileMetadata + if exciseSpan.Valid() && exciseSpan.Contains(d.cmp, m.Smallest) && exciseSpan.Contains(d.cmp, m.Largest) { + // This file fits perfectly within the excise span. We can slot it at + // L6, or sharedLevelsStart - 1 if we have shared files. + if len(lr.sharedMeta) > 0 { + f.Level = sharedLevelsStart - 1 + if baseLevel > f.Level { + f.Level = 0 + } + } else { + f.Level = 6 + } + } else { + // TODO(bilal): findTargetLevel does disk IO (reading files for data + // overlap) even though we're holding onto d.mu. Consider unlocking + // d.mu while we do this. We already hold versions.logLock so we should + // not see any version applications while we're at this. The one + // complication here would be pulling out the mu.compact.inProgress + // check from findTargetLevel, as that requires d.mu to be held. + f.Level, splitFile, err = findTargetLevel( + d.newIters, d.tableNewRangeKeyIter, iterOps, d.opts.Comparer, current, baseLevel, d.mu.compact.inProgress, m, shouldIngestSplit) + } + + if splitFile != nil { + if invariants.Enabled { + if lf := current.Levels[f.Level].Find(d.cmp, splitFile); lf == nil { + panic("splitFile returned is not in level it should be") + } + } + // We take advantage of the fact that we won't drop the db mutex + // between now and the call to logAndApply. So, no files should + // get added to a new in-progress compaction at this point. We can + // avoid having to iterate on in-progress compactions to cancel them + // if none of the files being split have a compacting state. + if splitFile.IsCompacting() { + checkCompactions = true + } + filesToSplit = append(filesToSplit, ingestSplitFile{ingestFile: m, splitFile: splitFile, level: f.Level}) + } + } if err != nil { d.mu.versions.logUnlock() return nil, err @@ -1111,26 +2120,159 @@ func (d *DB) ingestApply( levelMetrics.BytesIngested += m.Size levelMetrics.TablesIngested++ } + // replacedFiles maps files excised due to exciseSpan (or splitFiles returned + // by ingestTargetLevel), to files that were created to replace it. This map + // is used to resolve references to split files in filesToSplit, as it is + // possible for a file that we want to split to no longer exist or have a + // newer fileMetadata due to a split induced by another ingestion file, or an + // excise. + replacedFiles := make(map[base.FileNum][]newFileEntry) + updateLevelMetricsOnExcise := func(m *fileMetadata, level int, added []newFileEntry) { + levelMetrics := metrics[level] + if levelMetrics == nil { + levelMetrics = &LevelMetrics{} + metrics[level] = levelMetrics + } + levelMetrics.NumFiles-- + levelMetrics.Size -= int64(m.Size) + for i := range added { + levelMetrics.NumFiles++ + levelMetrics.Size += int64(added[i].Meta.Size) + } + } + if exciseSpan.Valid() { + // Iterate through all levels and find files that intersect with exciseSpan. + // + // TODO(bilal): We could drop the DB mutex here as we don't need it for + // excises; we only need to hold the version lock which we already are + // holding. However releasing the DB mutex could mess with the + // ingestTargetLevel calculation that happened above, as it assumed that it + // had a complete view of in-progress compactions that wouldn't change + // until logAndApply is called. If we were to drop the mutex now, we could + // schedule another in-progress compaction that would go into the chosen target + // level and lead to file overlap within level (which would panic in + // logAndApply). We should drop the db mutex here, do the excise, then + // re-grab the DB mutex and rerun just the in-progress compaction check to + // see if any new compactions are conflicting with our chosen target levels + // for files, and if they are, we should signal those compactions to error + // out. + for level := range current.Levels { + overlaps := current.Overlaps(level, d.cmp, exciseSpan.Start, exciseSpan.End, true /* exclusiveEnd */) + iter := overlaps.Iter() + + for m := iter.First(); m != nil; m = iter.Next() { + newFiles, err := d.excise(exciseSpan, m, ve, level) + if err != nil { + return nil, err + } + + if _, ok := ve.DeletedFiles[deletedFileEntry{ + Level: level, + FileNum: m.FileNum, + }]; !ok { + // We did not excise this file. + continue + } + replacedFiles[m.FileNum] = newFiles + updateLevelMetricsOnExcise(m, level, newFiles) + } + } + } + if len(filesToSplit) > 0 { + // For the same reasons as the above call to excise, we hold the db mutex + // while calling this method. + if err := d.ingestSplit(ve, updateLevelMetricsOnExcise, filesToSplit, replacedFiles); err != nil { + return nil, err + } + } + if len(filesToSplit) > 0 || exciseSpan.Valid() { + for c := range d.mu.compact.inProgress { + if c.versionEditApplied { + continue + } + // Check if this compaction overlaps with the excise span. Note that just + // checking if the inputs individually overlap with the excise span + // isn't sufficient; for instance, a compaction could have [a,b] and [e,f] + // as inputs and write it all out as [a,b,e,f] in one sstable. If we're + // doing a [c,d) excise at the same time as this compaction, we will have + // to error out the whole compaction as we can't guarantee it hasn't/won't + // write a file overlapping with the excise span. + if exciseSpan.OverlapsInternalKeyRange(d.cmp, c.smallest, c.largest) { + c.cancel.Store(true) + } + // Check if this compaction's inputs have been replaced due to an + // ingest-time split. In that case, cancel the compaction as a newly picked + // compaction would need to include any new files that slid in between + // previously-existing files. Note that we cancel any compaction that has a + // file that was ingest-split as an input, even if it started before this + // ingestion. + if checkCompactions { + for i := range c.inputs { + iter := c.inputs[i].files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { + if _, ok := replacedFiles[f.FileNum]; ok { + c.cancel.Store(true) + break + } + } + } + } + } + // Check for any EventuallyFileOnlySnapshots that could be watching for + // an excise on this span. + if exciseSpan.Valid() { + for s := d.mu.snapshots.root.next; s != &d.mu.snapshots.root; s = s.next { + if s.efos == nil { + continue + } + efos := s.efos + // TODO(bilal): We can make this faster by taking advantage of the sorted + // nature of protectedRanges to do a sort.Search, or even maintaining a + // global list of all protected ranges instead of having to peer into every + // snapshot. + for i := range efos.protectedRanges { + if efos.protectedRanges[i].OverlapsKeyRange(d.cmp, exciseSpan) { + efos.excised.Store(true) + break + } + } + } + } + } if err := d.mu.versions.logAndApply(jobID, ve, metrics, false /* forceRotation */, func() []compactionInfo { return d.getInProgressCompactionInfoLocked(nil) }); err != nil { return nil, err } + + d.mu.versions.metrics.Ingest.Count++ + d.updateReadStateLocked(d.opts.DebugCheck) + // updateReadStateLocked could have generated obsolete tables, schedule a + // cleanup job if necessary. + d.deleteObsoleteFiles(jobID) d.updateTableStatsLocked(ve.NewFiles) - d.deleteObsoleteFiles(jobID, false /* waitForOngoing */) // The ingestion may have pushed a level over the threshold for compaction, // so check to see if one is necessary and schedule it. d.maybeScheduleCompaction() - if scheduleFlush { - d.maybeScheduleFlush() + var toValidate []manifest.NewFileEntry + dedup := make(map[base.DiskFileNum]struct{}) + for _, entry := range ve.NewFiles { + if _, ok := dedup[entry.Meta.FileBacking.DiskFileNum]; !ok { + toValidate = append(toValidate, entry) + dedup[entry.Meta.FileBacking.DiskFileNum] = struct{}{} + } } - d.maybeValidateSSTablesLocked(ve.NewFiles) + d.maybeValidateSSTablesLocked(toValidate) return ve, nil } // maybeValidateSSTablesLocked adds the slice of newFileEntrys to the pending // queue of files to be validated, when the feature is enabled. +// +// Note that if two entries with the same backing file are added twice, then the +// block checksums for the backing file will be validated twice. +// // DB.mu must be locked when calling. func (d *DB) maybeValidateSSTablesLocked(newFiles []newFileEntry) { // Only add to the validation queue when the feature is enabled. @@ -1197,9 +2339,19 @@ func (d *DB) validateSSTables() { } } - err := d.tableCache.withReader(f.Meta, func(r *sstable.Reader) error { - return r.ValidateBlockChecksums() - }) + var err error + if f.Meta.Virtual { + err = d.tableCache.withVirtualReader( + f.Meta.VirtualMeta(), func(v sstable.VirtualReader) error { + return v.ValidateBlockChecksumsOnBacking() + }) + } else { + err = d.tableCache.withReader( + f.Meta.PhysicalMeta(), func(r *sstable.Reader) error { + return r.ValidateBlockChecksums() + }) + } + if err != nil { // TODO(travers): Hook into the corruption reporting pipeline, once // available. See pebble#1192. diff --git a/vendor/github.com/cockroachdb/pebble/internal.go b/vendor/github.com/cockroachdb/pebble/internal.go index 99fec08..61a4284 100644 --- a/vendor/github.com/cockroachdb/pebble/internal.go +++ b/vendor/github.com/cockroachdb/pebble/internal.go @@ -22,7 +22,8 @@ const ( InternalKeyKindRangeKeySet = base.InternalKeyKindRangeKeySet InternalKeyKindRangeKeyUnset = base.InternalKeyKindRangeKeyUnset InternalKeyKindRangeKeyDelete = base.InternalKeyKindRangeKeyDelete - InternalKeyKindIngestSST = base.InternalKeyIngestSST + InternalKeyKindIngestSST = base.InternalKeyKindIngestSST + InternalKeyKindDeleteSized = base.InternalKeyKindDeleteSized InternalKeyKindInvalid = base.InternalKeyKindInvalid InternalKeySeqNumBatch = base.InternalKeySeqNumBatch InternalKeySeqNumMax = base.InternalKeySeqNumMax diff --git a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go index ad0783d..011c3b0 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go +++ b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/arena.go @@ -18,22 +18,21 @@ package arenaskl import ( - "math" "sync/atomic" "unsafe" "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/constants" + "github.com/cockroachdb/pebble/internal/invariants" ) // Arena is lock-free. type Arena struct { - n uint64 + n atomic.Uint64 buf []byte } -const ( - align4 = 3 -) +const nodeAlignment = 4 var ( // ErrArenaFull indicates that the arena is full and cannot perform any more @@ -44,20 +43,28 @@ var ( // NewArena allocates a new arena using the specified buffer as the backing // store. func NewArena(buf []byte) *Arena { - // Don't store data at position 0 in order to reserve offset=0 as a kind - // of nil pointer. - return &Arena{ - n: 1, + if len(buf) > constants.MaxUint32OrInt { + if invariants.Enabled { + panic(errors.AssertionFailedf("attempting to create arena of size %d", len(buf))) + } + buf = buf[:constants.MaxUint32OrInt] + } + a := &Arena{ buf: buf, } + // We don't store data at position 0 in order to reserve offset=0 as a kind of + // nil pointer. + a.n.Store(1) + return a } // Size returns the number of bytes allocated by the arena. func (a *Arena) Size() uint32 { - s := atomic.LoadUint64(&a.n) - if s > math.MaxUint32 { - // Saturate at MaxUint32. - return math.MaxUint32 + s := a.n.Load() + if s > constants.MaxUint32OrInt { + // The last failed allocation can push the size higher than len(a.buf). + // Saturate at the maximum representable offset. + return constants.MaxUint32OrInt } return uint32(s) } @@ -67,24 +74,33 @@ func (a *Arena) Capacity() uint32 { return uint32(len(a.buf)) } -func (a *Arena) alloc(size, align, overflow uint32) (uint32, uint32, error) { +// alloc allocates a buffer of the given size and with the given alignment +// (which must be a power of 2). +// +// If overflow is not 0, it also ensures that many bytes after the buffer are +// inside the arena (this is used for structures that are larger than the +// requested size but don't use those extra bytes). +func (a *Arena) alloc(size, alignment, overflow uint32) (uint32, uint32, error) { + if invariants.Enabled && (alignment&(alignment-1)) != 0 { + panic(errors.AssertionFailedf("invalid alignment %d", alignment)) + } // Verify that the arena isn't already full. - origSize := atomic.LoadUint64(&a.n) + origSize := a.n.Load() if int(origSize) > len(a.buf) { return 0, 0, ErrArenaFull } // Pad the allocation with enough bytes to ensure the requested alignment. - padded := uint32(size) + align + padded := uint64(size) + uint64(alignment) - 1 - newSize := atomic.AddUint64(&a.n, uint64(padded)) - if int(newSize)+int(overflow) > len(a.buf) { + newSize := a.n.Add(padded) + if newSize+uint64(overflow) > uint64(len(a.buf)) { return 0, 0, ErrArenaFull } // Return the aligned offset. - offset := (uint32(newSize) - padded + align) & ^align - return offset, padded, nil + offset := (uint32(newSize) - size) & ^(alignment - 1) + return offset, uint32(padded), nil } func (a *Arena) getBytes(offset uint32, size uint32) []byte { diff --git a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go index 3a40a20..bad4909 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go +++ b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go @@ -18,7 +18,6 @@ package arenaskl import ( - "encoding/binary" "sync" "github.com/cockroachdb/pebble/internal/base" @@ -240,17 +239,8 @@ func (it *Iterator) SetBounds(lower, upper []byte) { } func (it *Iterator) decodeKey() { - b := it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize) - // This is a manual inline of base.DecodeInternalKey, because the Go compiler - // seems to refuse to automatically inline it currently. - l := len(b) - 8 - if l >= 0 { - it.key.Trailer = binary.LittleEndian.Uint64(b[l:]) - it.key.UserKey = b[:l:l] - } else { - it.key.Trailer = uint64(base.InternalKeyKindInvalid) - it.key.UserKey = nil - } + it.key.UserKey = it.list.arena.getBytes(it.nd.keyOffset, it.nd.keySize) + it.key.Trailer = it.nd.keyTrailer } func (it *Iterator) seekForBaseSplice(key []byte) (prev, next *node, found bool) { diff --git a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go index a605d9d..d464bc5 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go +++ b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/node.go @@ -29,25 +29,27 @@ import ( // is used here. If a key/value overflows a uint32, it should not be added to // the skiplist. func MaxNodeSize(keySize, valueSize uint32) uint64 { - return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + align4 + const maxPadding = nodeAlignment - 1 + return uint64(maxNodeSize) + uint64(keySize) + uint64(valueSize) + maxPadding } type links struct { - nextOffset uint32 - prevOffset uint32 + nextOffset atomic.Uint32 + prevOffset atomic.Uint32 } func (l *links) init(prevOffset, nextOffset uint32) { - l.nextOffset = nextOffset - l.prevOffset = prevOffset + l.nextOffset.Store(nextOffset) + l.prevOffset.Store(prevOffset) } type node struct { // Immutable fields, so no need to lock to access key. - keyOffset uint32 - keySize uint32 - valueSize uint32 - allocSize uint32 + keyOffset uint32 + keySize uint32 + keyTrailer uint64 + valueSize uint32 + allocSize uint32 // Most nodes do not need to use the full height of the tower, since the // probability of each successive level decreases exponentially. Because @@ -65,7 +67,7 @@ func newNode( if height < 1 || height > maxHeight { panic("height cannot be less than one or greater than the max height") } - keySize := key.Size() + keySize := len(key.UserKey) if int64(keySize) > math.MaxUint32 { panic("key is too large") } @@ -81,8 +83,8 @@ func newNode( if err != nil { return } - - key.Encode(nd.getKeyBytes(arena)) + nd.keyTrailer = key.Trailer + copy(nd.getKeyBytes(arena), key.UserKey) copy(nd.getValue(arena), value) return } @@ -93,7 +95,7 @@ func newRawNode(arena *Arena, height uint32, keySize, valueSize uint32) (nd *nod unusedSize := uint32((maxHeight - int(height)) * linksSize) nodeSize := uint32(maxNodeSize) - unusedSize - nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, align4, unusedSize) + nodeOffset, allocSize, err := arena.alloc(nodeSize+keySize+valueSize, nodeAlignment, unusedSize) if err != nil { return } @@ -115,17 +117,17 @@ func (n *node) getValue(arena *Arena) []byte { } func (n *node) nextOffset(h int) uint32 { - return atomic.LoadUint32(&n.tower[h].nextOffset) + return n.tower[h].nextOffset.Load() } func (n *node) prevOffset(h int) uint32 { - return atomic.LoadUint32(&n.tower[h].prevOffset) + return n.tower[h].prevOffset.Load() } func (n *node) casNextOffset(h int, old, val uint32) bool { - return atomic.CompareAndSwapUint32(&n.tower[h].nextOffset, old, val) + return n.tower[h].nextOffset.CompareAndSwap(old, val) } func (n *node) casPrevOffset(h int, old, val uint32) bool { - return atomic.CompareAndSwapUint32(&n.tower[h].prevOffset, old, val) + return n.tower[h].prevOffset.CompareAndSwap(old, val) } diff --git a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go index 3f29447..ef1ebfc 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go +++ b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go @@ -44,7 +44,6 @@ Key differences: package arenaskl // import "github.com/cockroachdb/pebble/internal/arenaskl" import ( - "encoding/binary" "math" "runtime" "sync/atomic" @@ -68,7 +67,7 @@ const ( // keys. var ErrRecordExists = errors.New("record with this key already exists") -// Skiplist is a fast, cocnurrent skiplist implementation that supports forward +// Skiplist is a fast, concurrent skiplist implementation that supports forward // and backward iteration. See batchskl.Skiplist for a non-concurrent // skiplist. Keys and values are immutable once added to the skiplist and // deletion is not supported. Instead, higher-level code is expected to add new @@ -80,7 +79,7 @@ type Skiplist struct { cmp base.Compare head *node tail *node - height uint32 // Current height. 1 <= height <= maxHeight. CAS. + height atomic.Uint32 // Current height. 1 <= height <= maxHeight. CAS. // If set to true by tests, then extra delays are added to make it easier to // detect unusual race conditions. @@ -140,22 +139,22 @@ func (s *Skiplist) Reset(arena *Arena, cmp base.Compare) { headOffset := arena.getPointerOffset(unsafe.Pointer(head)) tailOffset := arena.getPointerOffset(unsafe.Pointer(tail)) for i := 0; i < maxHeight; i++ { - head.tower[i].nextOffset = tailOffset - tail.tower[i].prevOffset = headOffset + head.tower[i].nextOffset.Store(tailOffset) + tail.tower[i].prevOffset.Store(headOffset) } *s = Skiplist{ - arena: arena, - cmp: cmp, - head: head, - tail: tail, - height: 1, + arena: arena, + cmp: cmp, + head: head, + tail: tail, } + s.height.Store(1) } // Height returns the height of the highest tower within any of the nodes that // have ever been allocated as part of this skiplist. -func (s *Skiplist) Height() uint32 { return atomic.LoadUint32(&s.height) } +func (s *Skiplist) Height() uint32 { return s.height.Load() } // Arena returns the arena backing this skiplist. func (s *Skiplist) Arena() *Arena { return s.arena } @@ -327,7 +326,7 @@ func (s *Skiplist) newNode( // Try to increase s.height via CAS. listHeight := s.Height() for height > listHeight { - if atomic.CompareAndSwapUint32(&s.height, listHeight, height) { + if s.height.CompareAndSwap(listHeight, height) { // Successfully increased skiplist.height. break } @@ -412,26 +411,19 @@ func (s *Skiplist) findSpliceForLevel( offset, size := next.keyOffset, next.keySize nextKey := s.arena.buf[offset : offset+size] - n := int32(size) - 8 - cmp := s.cmp(key.UserKey, nextKey[:n]) + cmp := s.cmp(key.UserKey, nextKey) if cmp < 0 { // We are done for this level, since prev.key < key < next.key. break } if cmp == 0 { // User-key equality. - var nextTrailer uint64 - if n >= 0 { - nextTrailer = binary.LittleEndian.Uint64(nextKey[n:]) - } else { - nextTrailer = uint64(base.InternalKeyKindInvalid) - } - if key.Trailer == nextTrailer { + if key.Trailer == next.keyTrailer { // Internal key equality. found = true break } - if key.Trailer > nextTrailer { + if key.Trailer > next.keyTrailer { // We are done for this level, since prev.key < key < next.key. break } @@ -446,8 +438,7 @@ func (s *Skiplist) findSpliceForLevel( func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool { ndKey := s.arena.buf[nd.keyOffset : nd.keyOffset+nd.keySize] - n := int32(nd.keySize) - 8 - cmp := s.cmp(ndKey[:n], key.UserKey) + cmp := s.cmp(ndKey, key.UserKey) if cmp < 0 { return true } @@ -455,25 +446,19 @@ func (s *Skiplist) keyIsAfterNode(nd *node, key base.InternalKey) bool { return false } // User-key equality. - var ndTrailer uint64 - if n >= 0 { - ndTrailer = binary.LittleEndian.Uint64(ndKey[n:]) - } else { - ndTrailer = uint64(base.InternalKeyKindInvalid) - } - if key.Trailer == ndTrailer { + if key.Trailer == nd.keyTrailer { // Internal key equality. return false } - return key.Trailer < ndTrailer + return key.Trailer < nd.keyTrailer } func (s *Skiplist) getNext(nd *node, h int) *node { - offset := atomic.LoadUint32(&nd.tower[h].nextOffset) + offset := nd.tower[h].nextOffset.Load() return (*node)(s.arena.getPointer(offset)) } func (s *Skiplist) getPrev(nd *node, h int) *node { - offset := atomic.LoadUint32(&nd.tower[h].prevOffset) + offset := nd.tower[h].prevOffset.Load() return (*node)(s.arena.getPointer(offset)) } diff --git a/vendor/github.com/cockroachdb/pebble/internal/base/filenames.go b/vendor/github.com/cockroachdb/pebble/internal/base/filenames.go index 269d14b..d2ba2f0 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/base/filenames.go +++ b/vendor/github.com/cockroachdb/pebble/internal/base/filenames.go @@ -20,6 +20,41 @@ type FileNum uint64 // String returns a string representation of the file number. func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) } +// SafeFormat implements redact.SafeFormatter. +func (fn FileNum) SafeFormat(w redact.SafePrinter, _ rune) { + w.Printf("%06d", redact.SafeUint(fn)) +} + +// DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be +// called if the caller can ensure that the FileNum belongs to a physical file +// on disk. These could be manifests, log files, physical sstables on disk, the +// options file, but not virtual sstables. +func (fn FileNum) DiskFileNum() DiskFileNum { + return DiskFileNum{fn} +} + +// A DiskFileNum is just a FileNum belonging to a file which exists on disk. +// Note that a FileNum is an internal DB identifier and it could belong to files +// which don't exist on disk. An example would be virtual sstable FileNums. +// Converting a DiskFileNum to a FileNum is always valid, whereas converting a +// FileNum to DiskFileNum may not be valid and care should be taken to prove +// that the FileNum actually exists on disk. +type DiskFileNum struct { + fn FileNum +} + +func (dfn DiskFileNum) String() string { return dfn.fn.String() } + +// SafeFormat implements redact.SafeFormatter. +func (dfn DiskFileNum) SafeFormat(w redact.SafePrinter, verb rune) { + dfn.fn.SafeFormat(w, verb) +} + +// FileNum converts a DiskFileNum to a FileNum. This conversion is always valid. +func (dfn DiskFileNum) FileNum() FileNum { + return dfn.fn +} + // FileType enumerates the types of files found in a DB. type FileType int @@ -36,92 +71,92 @@ const ( ) // MakeFilename builds a filename from components. -func MakeFilename(fileType FileType, fileNum FileNum) string { +func MakeFilename(fileType FileType, dfn DiskFileNum) string { switch fileType { case FileTypeLog: - return fmt.Sprintf("%s.log", fileNum) + return fmt.Sprintf("%s.log", dfn) case FileTypeLock: return "LOCK" case FileTypeTable: - return fmt.Sprintf("%s.sst", fileNum) + return fmt.Sprintf("%s.sst", dfn) case FileTypeManifest: - return fmt.Sprintf("MANIFEST-%s", fileNum) + return fmt.Sprintf("MANIFEST-%s", dfn) case FileTypeCurrent: return "CURRENT" case FileTypeOptions: - return fmt.Sprintf("OPTIONS-%s", fileNum) + return fmt.Sprintf("OPTIONS-%s", dfn) case FileTypeOldTemp: - return fmt.Sprintf("CURRENT.%s.dbtmp", fileNum) + return fmt.Sprintf("CURRENT.%s.dbtmp", dfn) case FileTypeTemp: - return fmt.Sprintf("temporary.%s.dbtmp", fileNum) + return fmt.Sprintf("temporary.%s.dbtmp", dfn) } panic("unreachable") } // MakeFilepath builds a filepath from components. -func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, fileNum FileNum) string { - return fs.PathJoin(dirname, MakeFilename(fileType, fileNum)) +func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, dfn DiskFileNum) string { + return fs.PathJoin(dirname, MakeFilename(fileType, dfn)) } // ParseFilename parses the components from a filename. -func ParseFilename(fs vfs.FS, filename string) (fileType FileType, fileNum FileNum, ok bool) { +func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileNum, ok bool) { filename = fs.PathBase(filename) switch { case filename == "CURRENT": - return FileTypeCurrent, 0, true + return FileTypeCurrent, DiskFileNum{0}, true case filename == "LOCK": - return FileTypeLock, 0, true + return FileTypeLock, DiskFileNum{0}, true case strings.HasPrefix(filename, "MANIFEST-"): - fileNum, ok = parseFileNum(filename[len("MANIFEST-"):]) + dfn, ok = parseDiskFileNum(filename[len("MANIFEST-"):]) if !ok { break } - return FileTypeManifest, fileNum, true + return FileTypeManifest, dfn, true case strings.HasPrefix(filename, "OPTIONS-"): - fileNum, ok = parseFileNum(filename[len("OPTIONS-"):]) + dfn, ok = parseDiskFileNum(filename[len("OPTIONS-"):]) if !ok { break } - return FileTypeOptions, fileNum, ok + return FileTypeOptions, dfn, ok case strings.HasPrefix(filename, "CURRENT.") && strings.HasSuffix(filename, ".dbtmp"): s := strings.TrimSuffix(filename[len("CURRENT."):], ".dbtmp") - fileNum, ok = parseFileNum(s) + dfn, ok = parseDiskFileNum(s) if !ok { break } - return FileTypeOldTemp, fileNum, ok + return FileTypeOldTemp, dfn, ok case strings.HasPrefix(filename, "temporary.") && strings.HasSuffix(filename, ".dbtmp"): s := strings.TrimSuffix(filename[len("temporary."):], ".dbtmp") - fileNum, ok = parseFileNum(s) + dfn, ok = parseDiskFileNum(s) if !ok { break } - return FileTypeTemp, fileNum, ok + return FileTypeTemp, dfn, ok default: i := strings.IndexByte(filename, '.') if i < 0 { break } - fileNum, ok = parseFileNum(filename[:i]) + dfn, ok = parseDiskFileNum(filename[:i]) if !ok { break } switch filename[i+1:] { case "sst": - return FileTypeTable, fileNum, true + return FileTypeTable, dfn, true case "log": - return FileTypeLog, fileNum, true + return FileTypeLog, dfn, true } } - return 0, fileNum, false + return 0, dfn, false } -func parseFileNum(s string) (fileNum FileNum, ok bool) { +func parseDiskFileNum(s string) (dfn DiskFileNum, ok bool) { u, err := strconv.ParseUint(s, 10, 64) if err != nil { - return fileNum, false + return dfn, false } - return FileNum(u), true + return DiskFileNum{FileNum(u)}, true } // A Fataler fatals a process with a message when called. diff --git a/vendor/github.com/cockroachdb/pebble/internal/base/internal.go b/vendor/github.com/cockroachdb/pebble/internal/base/internal.go index 208b88f..259ef98 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/base/internal.go +++ b/vendor/github.com/cockroachdb/pebble/internal/base/internal.go @@ -9,6 +9,17 @@ import ( "fmt" "strconv" "strings" + + "github.com/cockroachdb/redact" +) + +const ( + // SeqNumZero is the zero sequence number, set by compactions if they can + // guarantee there are no keys underneath an internal key. + SeqNumZero = uint64(0) + // SeqNumStart is the first sequence number assigned to a key. Sequence + // numbers 1-9 are reserved for potential future use. + SeqNumStart = uint64(10) ) // InternalKeyKind enumerates the kind of key: a deletion tombstone, a set @@ -24,6 +35,14 @@ const ( //InternalKeyKindColumnFamilyDeletion InternalKeyKind = 4 //InternalKeyKindColumnFamilyValue InternalKeyKind = 5 //InternalKeyKindColumnFamilyMerge InternalKeyKind = 6 + + // InternalKeyKindSingleDelete (SINGLEDEL) is a performance optimization + // solely for compactions (to reduce write amp and space amp). Readers other + // than compactions should treat SINGLEDEL as equivalent to a DEL. + // Historically, it was simpler for readers other than compactions to treat + // SINGLEDEL as equivalent to DEL, but as of the introduction of + // InternalKeyKindSSTableInternalObsoleteBit, this is also necessary for + // correctness. InternalKeyKindSingleDelete InternalKeyKind = 7 //InternalKeyKindColumnFamilySingleDelete InternalKeyKind = 8 //InternalKeyKindBeginPrepareXID InternalKeyKind = 9 @@ -60,14 +79,21 @@ const ( InternalKeyKindRangeKeyUnset InternalKeyKind = 20 InternalKeyKindRangeKeySet InternalKeyKind = 21 - // InternalKeyIngestSST is used to distinguish a batch that corresponds to + // InternalKeyKindIngestSST is used to distinguish a batch that corresponds to // the WAL entry for ingested sstables that are added to the flushable // queue. This InternalKeyKind cannot appear, amongst other key kinds in a // batch, or in an sstable. - InternalKeyIngestSST InternalKeyKind = 22 + InternalKeyKindIngestSST InternalKeyKind = 22 - // This maximum value isn't part of the file format. It's unlikely, - // but future extensions may increase this value. + // InternalKeyKindDeleteSized keys behave identically to + // InternalKeyKindDelete keys, except that they hold an associated uint64 + // value indicating the (len(key)+len(value)) of the shadowed entry the + // tombstone is expected to delete. This value is used to inform compaction + // heuristics, but is not required to be accurate for correctness. + InternalKeyKindDeleteSized InternalKeyKind = 23 + + // This maximum value isn't part of the file format. Future extensions may + // increase this value. // // When constructing an internal key to pass to DB.Seek{GE,LE}, // internalKeyComparer sorts decreasing by kind (after sorting increasing by @@ -75,14 +101,19 @@ const ( // which sorts 'less than or equal to' any other valid internalKeyKind, when // searching for any kind of internal key formed by a certain user key and // seqNum. - InternalKeyKindMax InternalKeyKind = 22 + InternalKeyKindMax InternalKeyKind = 23 + + // Internal to the sstable format. Not exposed by any sstable iterator. + // Declared here to prevent definition of valid key kinds that set this bit. + InternalKeyKindSSTableInternalObsoleteBit InternalKeyKind = 64 + InternalKeyKindSSTableInternalObsoleteMask InternalKeyKind = 191 // InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a // zero sequence number. - InternalKeyZeroSeqnumMaxTrailer = uint64(InternalKeyKindInvalid) + InternalKeyZeroSeqnumMaxTrailer = uint64(255) // A marker for an invalid key. - InternalKeyKindInvalid InternalKeyKind = 255 + InternalKeyKindInvalid InternalKeyKind = InternalKeyKindSSTableInternalObsoleteMask // InternalKeySeqNumBatch is a bit that is set on batch sequence numbers // which prevents those entries from being excluded from iteration. @@ -105,6 +136,9 @@ const ( InternalKeyBoundaryRangeKey = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeKeySet) ) +// Assert InternalKeyKindSSTableInternalObsoleteBit > InternalKeyKindMax +const _ = uint(InternalKeyKindSSTableInternalObsoleteBit - InternalKeyKindMax - 1) + var internalKeyKindNames = []string{ InternalKeyKindDelete: "DEL", InternalKeyKindSet: "SET", @@ -117,7 +151,8 @@ var internalKeyKindNames = []string{ InternalKeyKindRangeKeySet: "RANGEKEYSET", InternalKeyKindRangeKeyUnset: "RANGEKEYUNSET", InternalKeyKindRangeKeyDelete: "RANGEKEYDEL", - InternalKeyIngestSST: "INGESTSST", + InternalKeyKindIngestSST: "INGESTSST", + InternalKeyKindDeleteSized: "DELSIZED", InternalKeyKindInvalid: "INVALID", } @@ -128,6 +163,11 @@ func (k InternalKeyKind) String() string { return fmt.Sprintf("UNKNOWN:%d", k) } +// SafeFormat implements redact.SafeFormatter. +func (k InternalKeyKind) SafeFormat(w redact.SafePrinter, _ rune) { + w.Print(redact.SafeString(k.String())) +} + // InternalKey is a key used for the in-memory and on-disk partial DBs that // make up a pebble DB. // @@ -194,6 +234,7 @@ var kindsMap = map[string]InternalKeyKind{ "DEL": InternalKeyKindDelete, "SINGLEDEL": InternalKeyKindSingleDelete, "RANGEDEL": InternalKeyKindRangeDelete, + "LOGDATA": InternalKeyKindLogData, "SET": InternalKeyKindSet, "MERGE": InternalKeyKindMerge, "INVALID": InternalKeyKindInvalid, @@ -202,6 +243,8 @@ var kindsMap = map[string]InternalKeyKind{ "RANGEKEYSET": InternalKeyKindRangeKeySet, "RANGEKEYUNSET": InternalKeyKindRangeKeyUnset, "RANGEKEYDEL": InternalKeyKindRangeKeyDelete, + "INGESTSST": InternalKeyKindIngestSST, + "DELSIZED": InternalKeyKindDeleteSized, } // ParseInternalKey parses the string representation of an internal key. The @@ -338,6 +381,11 @@ func (k InternalKey) SeqNum() uint64 { return k.Trailer >> 8 } +// SeqNumFromTrailer returns the sequence number component of a trailer. +func SeqNumFromTrailer(t uint64) uint64 { + return t >> 8 +} + // Visible returns true if the key is visible at the specified snapshot // sequence number. func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool { @@ -395,6 +443,13 @@ func (k InternalKey) Clone() InternalKey { } } +// CopyFrom converts this InternalKey into a clone of the passed-in InternalKey, +// reusing any space already used for the current UserKey. +func (k *InternalKey) CopyFrom(k2 InternalKey) { + k.UserKey = append(k.UserKey[:0], k2.UserKey...) + k.Trailer = k2.Trailer +} + // String returns a string representation of the key. func (k InternalKey) String() string { return fmt.Sprintf("%s#%d,%d", FormatBytes(k.UserKey), k.SeqNum(), k.Kind()) diff --git a/vendor/github.com/cockroachdb/pebble/internal/base/metrics.go b/vendor/github.com/cockroachdb/pebble/internal/base/metrics.go index 4a7789a..520edc3 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/base/metrics.go +++ b/vendor/github.com/cockroachdb/pebble/internal/base/metrics.go @@ -53,6 +53,15 @@ func (tm *ThroughputMetric) Rate() int64 { float64(time.Second)) } +// Utilization returns a percent [0, 1.0] indicating the percent of time +// work was performed. +func (tm *ThroughputMetric) Utilization() float64 { + if tm.WorkDuration == 0 { + return 0 + } + return float64(tm.WorkDuration) / float64(tm.WorkDuration+tm.IdleDuration) +} + // GaugeSampleMetric is used to measure a gauge value (e.g. queue length) by // accumulating samples of that gauge. type GaugeSampleMetric struct { diff --git a/vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go b/vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go index 73bfa0e..f56d95c 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go +++ b/vendor/github.com/cockroachdb/pebble/internal/batchskl/skl.go @@ -63,14 +63,15 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/constants" "golang.org/x/exp/rand" ) const ( maxHeight = 20 - maxNodeSize = int(unsafe.Sizeof(node{})) - linksSize = int(unsafe.Sizeof(links{})) - maxNodesSize = math.MaxUint32 + maxNodeSize = uint64(unsafe.Sizeof(node{})) + linksSize = uint64(unsafe.Sizeof(links{})) + maxNodesSize = constants.MaxUint32OrInt ) var ( @@ -281,7 +282,7 @@ func (s *Skiplist) newNode( panic("height cannot be less than one or greater than the max height") } - unusedSize := (maxHeight - int(height)) * linksSize + unusedSize := uint64(maxHeight-int(height)) * linksSize nodeOffset, err := s.alloc(uint32(maxNodeSize - unusedSize)) if err != nil { return 0, err @@ -296,13 +297,13 @@ func (s *Skiplist) newNode( } func (s *Skiplist) alloc(size uint32) (uint32, error) { - offset := len(s.nodes) + offset := uint64(len(s.nodes)) // We only have a need for memory up to offset + size, but we never want // to allocate a node whose tail points into unallocated memory. minAllocSize := offset + maxNodeSize - if cap(s.nodes) < minAllocSize { - allocSize := cap(s.nodes) * 2 + if uint64(cap(s.nodes)) < minAllocSize { + allocSize := uint64(cap(s.nodes)) * 2 if allocSize < minAllocSize { allocSize = minAllocSize } diff --git a/vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go b/vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go index de6b954..cdae6a9 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go +++ b/vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go @@ -33,7 +33,7 @@ import ( type fileKey struct { // id is the namespace for fileNums. id uint64 - fileNum base.FileNum + fileNum base.DiskFileNum } type key struct { @@ -71,14 +71,12 @@ func (h Handle) Get() []byte { // Release releases the reference to the cache entry. func (h Handle) Release() { - if h.value != nil { - h.value.release() - } + h.value.release() } type shard struct { - hits int64 - misses int64 + hits atomic.Int64 + misses atomic.Int64 mu sync.RWMutex @@ -113,25 +111,25 @@ type shard struct { countTest int64 } -func (c *shard) Get(id uint64, fileNum base.FileNum, offset uint64) Handle { +func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle { c.mu.RLock() var value *Value if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil { value = e.acquireValue() if value != nil { - atomic.StoreInt32(&e.referenced, 1) + e.referenced.Store(true) } } c.mu.RUnlock() if value == nil { - atomic.AddInt64(&c.misses, 1) + c.misses.Add(1) return Handle{} } - atomic.AddInt64(&c.hits, 1) + c.hits.Add(1) return Handle{value: value} } -func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle { +func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle { if n := value.refs(); n != 1 { panic(fmt.Sprintf("pebble: Value has already been added to the cache: refs=%d", n)) } @@ -160,7 +158,7 @@ func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value case e.peekValue() != nil: // cache entry was a hot or cold page e.setValue(value) - atomic.StoreInt32(&e.referenced, 1) + e.referenced.Store(true) delta := int64(len(value.buf)) - e.size e.size = int64(len(value.buf)) if e.ptype == etHot { @@ -176,7 +174,7 @@ func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value // cache entry was a test page c.sizeTest -= e.size c.countTest-- - c.metaDel(e) + c.metaDel(e).release() c.metaCheck(e) e.size = int64(len(value.buf)) @@ -185,7 +183,7 @@ func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value c.coldTarget = c.targetSize() } - atomic.StoreInt32(&e.referenced, 0) + e.referenced.Store(false) e.setValue(value) e.ptype = etHot if c.metaAdd(k, e) { @@ -222,7 +220,7 @@ func (c *shard) checkConsistency() { } // Delete deletes the cached value for the specified file and offset. -func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) { +func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) { // The common case is there is nothing to delete, so do a quick check with // shared lock. k := key{fileKey{id, fileNum}, offset} @@ -233,37 +231,75 @@ func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) { return } - c.mu.Lock() - defer c.mu.Unlock() + var deletedValue *Value + func() { + c.mu.Lock() + defer c.mu.Unlock() - e := c.blocks.Get(k) - if e == nil { - return - } - c.metaEvict(e) - - c.checkConsistency() + e := c.blocks.Get(k) + if e == nil { + return + } + deletedValue = c.metaEvict(e) + c.checkConsistency() + }() + // Now that the mutex has been dropped, release the reference which will + // potentially free the memory associated with the previous cached value. + deletedValue.release() } // EvictFile evicts all of the cache values for the specified file. -func (c *shard) EvictFile(id uint64, fileNum base.FileNum) { - c.mu.Lock() - defer c.mu.Unlock() - +func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) { fkey := key{fileKey{id, fileNum}, 0} + for c.evictFileRun(fkey) { + // Sched switch to give another goroutine an opportunity to acquire the + // shard mutex. + runtime.Gosched() + } +} + +func (c *shard) evictFileRun(fkey key) (moreRemaining bool) { + // If most of the file's blocks are held in the block cache, evicting all + // the blocks may take a while. We don't want to block the entire cache + // shard, forcing concurrent readers to wait until we're finished. We drop + // the mutex every [blocksPerMutexAcquisition] blocks to give other + // goroutines an opportunity to make progress. + const blocksPerMutexAcquisition = 5 + c.mu.Lock() + + // Releasing a value may result in free-ing it back to the memory allocator. + // This can have a nontrivial cost that we'd prefer to not pay while holding + // the shard mutex, so we collect the evicted values in a local slice and + // only release them in a defer after dropping the cache mutex. + var obsoleteValuesAlloc [blocksPerMutexAcquisition]*Value + obsoleteValues := obsoleteValuesAlloc[:0] + defer func() { + c.mu.Unlock() + for _, v := range obsoleteValues { + v.release() + } + }() + blocks := c.files.Get(fkey) if blocks == nil { - return - } - for b, n := blocks, (*entry)(nil); ; b = n { - n = b.fileLink.next - c.metaEvict(b) - if b == n { - break - } + // No blocks for this file. + return false } - c.checkConsistency() + // b is the current head of the doubly linked list, and n is the entry after b. + for b, n := blocks, (*entry)(nil); len(obsoleteValues) < cap(obsoleteValues); b = n { + n = b.fileLink.next + obsoleteValues = append(obsoleteValues, c.metaEvict(b)) + if b == n { + // b == n represents the case where b was the last entry remaining + // in the doubly linked list, which is why it pointed at itself. So + // no more entries left. + c.checkConsistency() + return false + } + } + // Exhausted blocksPerMutexAcquisition. + return true } func (c *shard) Free() { @@ -274,7 +310,7 @@ func (c *shard) Free() { // metaCheck call when the "invariants" build tag is specified. for c.handHot != nil { e := c.handHot - c.metaDel(c.handHot) + c.metaDel(c.handHot).release() e.free() } @@ -359,12 +395,14 @@ func (c *shard) metaAdd(key key, e *entry) bool { // Remove the entry from the cache. This removes the entry from the blocks map, // the files map, and ensures that hand{Hot,Cold,Test} are not pointing at the -// entry. -func (c *shard) metaDel(e *entry) { +// entry. Returns the deleted value that must be released, if any. +func (c *shard) metaDel(e *entry) (deletedValue *Value) { if value := e.peekValue(); value != nil { value.ref.trace("metaDel") } - e.setValue(nil) + // Remove the pointer to the value. + deletedValue = e.val + e.val = nil c.blocks.Delete(e.key) if entriesGoAllocated { @@ -396,6 +434,7 @@ func (c *shard) metaDel(e *entry) { } else { c.files.Put(fkey, next) } + return deletedValue } // Check that the specified entry is not referenced by the cache. @@ -455,7 +494,7 @@ func (c *shard) metaCheck(e *entry) { } } -func (c *shard) metaEvict(e *entry) { +func (c *shard) metaEvict(e *entry) (evictedValue *Value) { switch e.ptype { case etHot: c.sizeHot -= e.size @@ -467,9 +506,10 @@ func (c *shard) metaEvict(e *entry) { c.sizeTest -= e.size c.countTest-- } - c.metaDel(e) + evictedValue = c.metaDel(e) c.metaCheck(e) e.free() + return evictedValue } func (c *shard) evict() { @@ -491,8 +531,8 @@ func (c *shard) runHandCold(countColdDebug, sizeColdDebug int64) { e := c.handCold if e.ptype == etCold { - if atomic.LoadInt32(&e.referenced) == 1 { - atomic.StoreInt32(&e.referenced, 0) + if e.referenced.Load() { + e.referenced.Store(false) e.ptype = etHot c.sizeCold -= e.size c.countCold-- @@ -528,8 +568,8 @@ func (c *shard) runHandHot() { e := c.handHot if e.ptype == etHot { - if atomic.LoadInt32(&e.referenced) == 1 { - atomic.StoreInt32(&e.referenced, 0) + if e.referenced.Load() { + e.referenced.Store(false) } else { e.ptype = etCold c.sizeHot -= e.size @@ -564,7 +604,7 @@ func (c *shard) runHandTest() { if c.coldTarget < 0 { c.coldTarget = 0 } - c.metaDel(e) + c.metaDel(e).release() c.metaCheck(e) e.free() } @@ -587,7 +627,7 @@ type Metrics struct { // Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is // used for page replacement // (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In -// order to provide better concurrency, 2 x NumCPUs shards are created, with +// order to provide better concurrency, 4 x NumCPUs shards are created, with // each shard being given 1/n of the target cache size. The Clock-PRO algorithm // is run independently on each shard. // @@ -621,9 +661,9 @@ type Metrics struct { // used in combination by specifying `-tags invariants,tracing`. Note that // "tracing" produces a significant slowdown, while "invariants" does not. type Cache struct { - refs int64 + refs atomic.Int64 maxSize int64 - idAlloc uint64 + idAlloc atomic.Uint64 shards []shard // Traces recorded by Cache.trace. Used for debugging. @@ -643,17 +683,44 @@ type Cache struct { // defer c.Unref() // d, err := pebble.Open(pebble.Options{Cache: c}) func New(size int64) *Cache { - return newShards(size, 2*runtime.GOMAXPROCS(0)) + // How many cache shards should we create? + // + // Note that the probability two processors will try to access the same + // shard at the same time increases superlinearly with the number of + // processors (Eg, consider the brithday problem where each CPU is a person, + // and each shard is a possible birthday). + // + // We could consider growing the number of shards superlinearly, but + // increasing the shard count may reduce the effectiveness of the caching + // algorithm if frequently-accessed blocks are insufficiently distributed + // across shards. If a shard's size is smaller than a single frequently + // scanned sstable, then the shard will be unable to hold the entire + // frequently-scanned table in memory despite other shards still holding + // infrequently accessed blocks. + // + // Experimentally, we've observed contention contributing to tail latencies + // at 2 shards per processor. For now we use 4 shards per processor, + // recognizing this may not be final word. + m := 4 * runtime.GOMAXPROCS(0) + + // In tests we can use large CPU machines with small cache sizes and have + // many caches in existence at a time. If sharding into m shards would + // produce too small shards, constrain the number of shards to 4. + const minimumShardSize = 4 << 20 // 4 MiB + if m > 4 && int(size)/m < minimumShardSize { + m = 4 + } + return newShards(size, m) } func newShards(size int64, shards int) *Cache { c := &Cache{ - refs: 1, maxSize: size, - idAlloc: 1, shards: make([]shard, shards), } - c.trace("alloc", c.refs) + c.refs.Store(1) + c.idAlloc.Store(1) + c.trace("alloc", c.refs.Load()) for i := range c.shards { c.shards[i] = shard{ maxSize: size / int64(len(c.shards)), @@ -669,7 +736,7 @@ func newShards(size int64, shards int) *Cache { // Note: this is a no-op if invariants are disabled or race is enabled. invariants.SetFinalizer(c, func(obj interface{}) { c := obj.(*Cache) - if v := atomic.LoadInt64(&c.refs); v != 0 { + if v := c.refs.Load(); v != 0 { c.tr.Lock() fmt.Fprintf(os.Stderr, "pebble: cache (%p) has non-zero reference count: %d\n", c, v) @@ -683,7 +750,7 @@ func newShards(size int64, shards int) *Cache { return c } -func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard { +func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *shard { if id == 0 { panic("pebble: 0 cache ID is invalid") } @@ -698,10 +765,11 @@ func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard h ^= uint64(id & 0xff) id >>= 8 } + fileNumVal := uint64(fileNum.FileNum()) for i := 0; i < 8; i++ { h *= prime64 - h ^= uint64(fileNum & 0xff) - fileNum >>= 8 + h ^= uint64(fileNumVal) & 0xff + fileNumVal >>= 8 } for i := 0; i < 8; i++ { h *= prime64 @@ -715,7 +783,7 @@ func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard // Ref adds a reference to the cache. The cache only remains valid as long a // reference is maintained to it. func (c *Cache) Ref() { - v := atomic.AddInt64(&c.refs, 1) + v := c.refs.Add(1) if v <= 1 { panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v)) } @@ -724,7 +792,7 @@ func (c *Cache) Ref() { // Unref releases a reference on the cache. func (c *Cache) Unref() { - v := atomic.AddInt64(&c.refs, -1) + v := c.refs.Add(-1) c.trace("unref", v) switch { case v < 0: @@ -738,7 +806,7 @@ func (c *Cache) Unref() { // Get retrieves the cache value for the specified file and offset, returning // nil if no value is present. -func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle { +func (c *Cache) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle { return c.getShard(id, fileNum, offset).Get(id, fileNum, offset) } @@ -746,17 +814,17 @@ func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle { // existing value if present. A Handle is returned which provides faster // retrieval of the cached value than Get (lock-free and avoidance of the map // lookup). The value must have been allocated by Cache.Alloc. -func (c *Cache) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle { +func (c *Cache) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle { return c.getShard(id, fileNum, offset).Set(id, fileNum, offset, value) } // Delete deletes the cached value for the specified file and offset. -func (c *Cache) Delete(id uint64, fileNum base.FileNum, offset uint64) { +func (c *Cache) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) { c.getShard(id, fileNum, offset).Delete(id, fileNum, offset) } // EvictFile evicts all of the cache values for the specified file. -func (c *Cache) EvictFile(id uint64, fileNum base.FileNum) { +func (c *Cache) EvictFile(id uint64, fileNum base.DiskFileNum) { if id == 0 { panic("pebble: 0 cache ID is invalid") } @@ -784,14 +852,14 @@ func (c *Cache) Size() int64 { // manually managed. The caller MUST either add the value to the cache (via // Cache.Set), or release the value (via Cache.Free). Failure to do so will // result in a memory leak. -func (c *Cache) Alloc(n int) *Value { +func Alloc(n int) *Value { return newValue(n) } // Free frees the specified value. The buffer associated with the value will // possibly be reused, making it invalid to use the buffer after calling // Free. Do not call Free on a value that has been added to the cache. -func (c *Cache) Free(v *Value) { +func Free(v *Value) { if n := v.refs(); n > 1 { panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n)) } @@ -828,8 +896,8 @@ func (c *Cache) Metrics() Metrics { m.Count += int64(s.blocks.Count()) m.Size += s.sizeHot + s.sizeCold s.mu.RUnlock() - m.Hits += atomic.LoadInt64(&s.hits) - m.Misses += atomic.LoadInt64(&s.misses) + m.Hits += s.hits.Load() + m.Misses += s.misses.Load() } return m } @@ -837,5 +905,5 @@ func (c *Cache) Metrics() Metrics { // NewID returns a new ID to be used as a namespace for cached file // blocks. func (c *Cache) NewID() uint64 { - return atomic.AddUint64(&c.idAlloc, 1) + return c.idAlloc.Add(1) } diff --git a/vendor/github.com/cockroachdb/pebble/internal/cache/entry.go b/vendor/github.com/cockroachdb/pebble/internal/cache/entry.go index beb4891..a49fde6 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/cache/entry.go +++ b/vendor/github.com/cockroachdb/pebble/internal/cache/entry.go @@ -4,6 +4,8 @@ package cache +import "sync/atomic" + type entryType int8 const ( @@ -56,7 +58,7 @@ type entry struct { ptype entryType // referenced is atomically set to indicate that this entry has been accessed // since the last time one of the clock hands swept it. - referenced int32 + referenced atomic.Bool shard *shard // Reference count for the entry. The entry is freed when the reference count // drops to zero. @@ -137,9 +139,7 @@ func (e *entry) setValue(v *Value) { } old := e.val e.val = v - if old != nil { - old.release() - } + old.release() } func (e *entry) peekValue() *Value { diff --git a/vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go b/vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go index eb2c886..6e093fd 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go +++ b/vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go @@ -24,7 +24,7 @@ func robinHoodHash(k key, shift uint32) uint32 { const m = 11400714819323198485 h := hashSeed h ^= k.id * m - h ^= uint64(k.fileNum) * m + h ^= uint64(k.fileNum.FileNum()) * m h ^= k.offset * m return uint32(h >> shift) } diff --git a/vendor/github.com/cockroachdb/pebble/internal/cache/value.go b/vendor/github.com/cockroachdb/pebble/internal/cache/value.go index c5c967a..6d2cae1 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/cache/value.go +++ b/vendor/github.com/cockroachdb/pebble/internal/cache/value.go @@ -40,7 +40,7 @@ func (v *Value) acquire() { } func (v *Value) release() { - if v.ref.release() { + if v != nil && v.ref.release() { v.free() } } diff --git a/vendor/github.com/cockroachdb/pebble/internal/constants/constants.go b/vendor/github.com/cockroachdb/pebble/internal/constants/constants.go new file mode 100644 index 0000000..8d9198c --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/internal/constants/constants.go @@ -0,0 +1,17 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package constants + +const ( + // oneIf64Bit is 1 on 64-bit platforms and 0 on 32-bit platforms. + oneIf64Bit = ^uint(0) >> 63 + + // MaxUint32OrInt returns min(MaxUint32, MaxInt), i.e + // - MaxUint32 on 64-bit platforms; + // - MaxInt on 32-bit platforms. + // It is used when slices are limited to Uint32 on 64-bit platforms (the + // length limit for slices is naturally MaxInt on 32-bit platforms). + MaxUint32OrInt = (1<<31)<= key, _if_ the filterFunc (which has + // no knowledge of the seek key) mutated the span to start at a key greater + // than or equal to `key`. Detect this case and prev/invalidate the iter. + if span != nil && i.cmp(span.Start, key) >= 0 { + return i.Prev() + } + return span } // First implements FragmentIterator. diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go index a507b75..6419821 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go @@ -7,6 +7,7 @@ package keyspan import ( "fmt" + "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/invariants" ) @@ -111,6 +112,28 @@ type InterleavingIter struct { nextPrefixBuf []byte pointKey *base.InternalKey pointVal base.LazyValue + // err holds an iterator error from either pointIter or keyspanIter. It's + // reset to nil on seeks. An overview of error-handling mechanics: + // + // Whenever either pointIter or keyspanIter is respositioned and a nil + // key/span is returned, the code performing the positioning is responsible + // for checking the iterator's Error() value. This happens in savePoint and + // saveSpan[Forward,Backward]. + // + // Once i.err is non-nil, the computation of i.pos must set i.pos = + // posExhausted. This happens in compute[Smallest|Largest]Pos and + // [next|prev]Pos. Setting i.pos to posExhausted ensures we'll yield nil to + // the caller, which they'll interpret as a signal they must check Error(). + // + // INVARIANTS: + // i.err != nil => i.pos = posExhausted + err error + // prefix records the iterator's current prefix if the iterator is in prefix + // mode. During prefix mode, Pebble will truncate spans to the next prefix. + // If the iterator subsequently leaves prefix mode, the existing span cached + // in i.span must be invalidated because its bounds do not reflect the + // original span's true bounds. + prefix []byte // span holds the span at the keyspanIter's current position. If the span is // wholly contained within the iterator bounds, this span is directly // returned to the iterator consumer through Span(). If either bound needed @@ -126,31 +149,16 @@ type InterleavingIter struct { truncatedSpan Span truncated bool - // Keeping all of the bools together reduces the sizeof the struct. + // Keeping all of the bools/uint8s together reduces the sizeof the struct. - // spanCoversKey indicates whether the current span covers the last-returned - // key. - spanCoversKey bool - // pointKeyInterleaved indicates whether the current point key has been - // interleaved in the current direction. - pointKeyInterleaved bool - // keyspanInterleaved indicates whether or not the current span has been - // interleaved at its start key in the current direction. A span marker is - // interleaved when first passing over the start key. - // - // When iterating in the forward direction, the span start key is - // interleaved when the span first begins to cover the current iterator - // position. The keyspan iterator isn't advanced until the - // InterleavingIterator moves beyond the current span's end key. This field - // is used to remember that the span has already been interleaved and - // shouldn't be interleaved again. - // - // When iterating in the reverse direction, the span start key is - // interleaved immediately before the iterator will move to a key no longer - // be covered by the span. This field behaves analagously to - // pointKeyInterleaved and if true signals that we must Prev the keyspan - // iterator on the next Prev call. - keyspanInterleaved bool + // pos encodes the current position of the iterator: exhausted, on the point + // key, on a keyspan start, or on a keyspan end. + pos interleavePos + // withinSpan indicates whether the iterator is currently positioned within + // the bounds of the current span (i.span). withinSpan must be updated + // whenever the interleaving iterator's position enters or exits the bounds + // of a span. + withinSpan bool // spanMarkerTruncated is set by SeekGE/SeekPrefixGE calls that truncate a // span's start bound marker to the search key. It's returned to false on // the next repositioning of the keyspan iterator. @@ -158,39 +166,54 @@ type InterleavingIter struct { // maskSpanChangedCalled records whether or not the last call to // SpanMask.SpanChanged provided the current span (i.span) or not. maskSpanChangedCalled bool - // prefix records whether the iteator is in prefix mode. During prefix mode, - // Pebble will truncate spans to the next prefix. If the iterator - // subsequently leaves prefix mode, the existing span cached in i.span must - // be invalidated because its bounds do not reflect the original span's true - // bounds. - prefix bool // dir indicates the direction of iteration: forward (+1) or backward (-1) dir int8 } +// interleavePos indicates the iterator's current position. Note that both +// keyspanStart and keyspanEnd positions correspond to their user key boundaries +// with maximal sequence numbers. This means in the forward direction +// posKeyspanStart and posKeyspanEnd are always interleaved before a posPointKey +// with the same user key. +type interleavePos int8 + +const ( + posUninitialized interleavePos = iota + posExhausted + posPointKey + posKeyspanStart + posKeyspanEnd +) + // Assert that *InterleavingIter implements the InternalIterator interface. var _ base.InternalIterator = &InterleavingIter{} +// InterleavingIterOpts holds options configuring the behavior of a +// InterleavingIter. +type InterleavingIterOpts struct { + Mask SpanMask + LowerBound, UpperBound []byte +} + // Init initializes the InterleavingIter to interleave point keys from pointIter // with key spans from keyspanIter. // -// The point iterator must already have the provided bounds. Init does not -// propagate the bounds down the iterator stack. +// The point iterator must already have the bounds provided on opts. Init does +// not propagate the bounds down the iterator stack. func (i *InterleavingIter) Init( comparer *base.Comparer, pointIter base.InternalIterator, keyspanIter FragmentIterator, - mask SpanMask, - lowerBound, upperBound []byte, + opts InterleavingIterOpts, ) { *i = InterleavingIter{ cmp: comparer.Compare, comparer: comparer, pointIter: pointIter, keyspanIter: keyspanIter, - mask: mask, - lower: lowerBound, - upper: upperBound, + mask: opts.Mask, + lower: opts.LowerBound, + upper: opts.UpperBound, } } @@ -209,14 +232,14 @@ func (i *InterleavingIter) InitSeekGE( ) (*base.InternalKey, base.LazyValue) { i.dir = +1 i.clearMask() - i.prefix = prefix != nil - i.pointKey, i.pointVal = pointKey, pointValue - i.pointKeyInterleaved = false + i.prefix = prefix + i.savePoint(pointKey, pointValue) // NB: This keyspanSeekGE call will truncate the span to the seek key if // necessary. This truncation is important for cases where a switch to // combined iteration is made during a user-initiated SeekGE. i.keyspanSeekGE(key, prefix) - return i.interleaveForward(key, prefix) + i.computeSmallestPos() + return i.yieldPosition(key, i.nextPos) } // InitSeekLT may be called after Init but before any positioning method. @@ -234,10 +257,10 @@ func (i *InterleavingIter) InitSeekLT( ) (*base.InternalKey, base.LazyValue) { i.dir = -1 i.clearMask() - i.pointKey, i.pointVal = pointKey, pointValue - i.pointKeyInterleaved = false + i.savePoint(pointKey, pointValue) i.keyspanSeekLT(key) - return i.interleaveBackward() + i.computeLargestPos() + return i.yieldPosition(i.lower, i.prevPos) } // SeekGE implements (base.InternalIterator).SeekGE. @@ -254,10 +277,10 @@ func (i *InterleavingIter) InitSeekLT( func (i *InterleavingIter) SeekGE( key []byte, flags base.SeekGEFlags, ) (*base.InternalKey, base.LazyValue) { + i.err = nil i.clearMask() i.disablePrefixMode() - i.pointKey, i.pointVal = i.pointIter.SeekGE(key, flags) - i.pointKeyInterleaved = false + i.savePoint(i.pointIter.SeekGE(key, flags)) // We need to seek the keyspan iterator too. If the keyspan iterator was // already positioned at a span, we might be able to avoid the seek if the @@ -265,14 +288,15 @@ func (i *InterleavingIter) SeekGE( if i.span != nil && i.cmp(key, i.span.End) < 0 && i.cmp(key, i.span.Start) >= 0 { // We're seeking within the existing span's bounds. We still might need // truncate the span to the iterator's bounds. - i.checkForwardBound(nil /* prefix */) + i.saveSpanForward(i.span) i.savedKeyspan() } else { i.keyspanSeekGE(key, nil /* prefix */) } i.dir = +1 - return i.interleaveForward(key, nil /* prefix */) + i.computeSmallestPos() + return i.yieldPosition(key, i.nextPos) } // SeekPrefixGE implements (base.InternalIterator).SeekPrefixGE. @@ -289,10 +313,10 @@ func (i *InterleavingIter) SeekGE( func (i *InterleavingIter) SeekPrefixGE( prefix, key []byte, flags base.SeekGEFlags, ) (*base.InternalKey, base.LazyValue) { + i.err = nil i.clearMask() - i.pointKey, i.pointVal = i.pointIter.SeekPrefixGE(prefix, key, flags) - i.pointKeyInterleaved = false - i.prefix = true + i.prefix = prefix + i.savePoint(i.pointIter.SeekPrefixGE(prefix, key, flags)) // We need to seek the keyspan iterator too. If the keyspan iterator was // already positioned at a span, we might be able to avoid the seek if the @@ -321,7 +345,7 @@ func (i *InterleavingIter) SeekPrefixGE( if ei := i.comparer.Split(i.span.End); i.cmp(prefix, i.span.End[:ei]) < 0 { // We're seeking within the existing span's bounds. We still might need // truncate the span to the iterator's bounds. - i.checkForwardBound(prefix) + i.saveSpanForward(i.span) i.savedKeyspan() seekKeyspanIter = false } @@ -331,17 +355,18 @@ func (i *InterleavingIter) SeekPrefixGE( } i.dir = +1 - return i.interleaveForward(key, prefix) + i.computeSmallestPos() + return i.yieldPosition(key, i.nextPos) } // SeekLT implements (base.InternalIterator).SeekLT. func (i *InterleavingIter) SeekLT( key []byte, flags base.SeekLTFlags, ) (*base.InternalKey, base.LazyValue) { + i.err = nil i.clearMask() i.disablePrefixMode() - i.pointKey, i.pointVal = i.pointIter.SeekLT(key, flags) - i.pointKeyInterleaved = false + i.savePoint(i.pointIter.SeekLT(key, flags)) // We need to seek the keyspan iterator too. If the keyspan iterator was // already positioned at a span, we might be able to avoid the seek if the @@ -349,13 +374,13 @@ func (i *InterleavingIter) SeekLT( if i.span != nil && i.cmp(key, i.span.Start) > 0 && i.cmp(key, i.span.End) < 0 { // We're seeking within the existing span's bounds. We still might need // truncate the span to the iterator's bounds. - i.checkBackwardBound() + i.saveSpanBackward(i.span) // The span's start key is still not guaranteed to be less than key, // because of the bounds enforcement. Consider the following example: // // Bounds are set to [d,e). The user performs a SeekLT(d). The // FragmentIterator.SeekLT lands on a span [b,f). This span has a start - // key less than d, as expected. Above, checkBackwardBound truncates the + // key less than d, as expected. Above, saveSpanBackward truncates the // span to match the iterator's current bounds, modifying the span to // [d,e), which does not overlap the search space of [-∞, d). // @@ -371,33 +396,34 @@ func (i *InterleavingIter) SeekLT( } i.dir = -1 - return i.interleaveBackward() + i.computeLargestPos() + return i.yieldPosition(i.lower, i.prevPos) } // First implements (base.InternalIterator).First. func (i *InterleavingIter) First() (*base.InternalKey, base.LazyValue) { + i.err = nil i.clearMask() i.disablePrefixMode() - i.pointKey, i.pointVal = i.pointIter.First() - i.pointKeyInterleaved = false - i.span = i.keyspanIter.First() - i.checkForwardBound(nil /* prefix */) + i.savePoint(i.pointIter.First()) + i.saveSpanForward(i.keyspanIter.First()) i.savedKeyspan() i.dir = +1 - return i.interleaveForward(i.lower, nil /* prefix */) + i.computeSmallestPos() + return i.yieldPosition(i.lower, i.nextPos) } // Last implements (base.InternalIterator).Last. func (i *InterleavingIter) Last() (*base.InternalKey, base.LazyValue) { + i.err = nil i.clearMask() i.disablePrefixMode() - i.pointKey, i.pointVal = i.pointIter.Last() - i.pointKeyInterleaved = false - i.span = i.keyspanIter.Last() - i.checkBackwardBound() + i.savePoint(i.pointIter.Last()) + i.saveSpanBackward(i.keyspanIter.Last()) i.savedKeyspan() i.dir = -1 - return i.interleaveBackward() + i.computeLargestPos() + return i.yieldPosition(i.lower, i.prevPos) } // Next implements (base.InternalIterator).Next. @@ -413,70 +439,42 @@ func (i *InterleavingIter) Next() (*base.InternalKey, base.LazyValue) { i.clearMask() } - // The existing point key (denoted below with *) is either the last - // key we returned (the current iterator position): - // points: x (y*) z - // or the upcoming point key in the backward direction if we just - // returned a span start boundary key: - // points: x* z - // spans: ([y-?)) - // direction. Either way, we must move to the next point key. - switch { - case i.pointKey == nil && i.lower == nil: - i.pointKey, i.pointVal = i.pointIter.First() - case i.pointKey == nil && i.lower != nil: - i.pointKey, i.pointVal = i.pointIter.SeekGE(i.lower, base.SeekGEFlagsNone) - default: - i.pointKey, i.pointVal = i.pointIter.Next() - } - i.pointKeyInterleaved = false - - if i.span == nil { - // There was no span in the reverse direction, but there may be - // a span in the forward direction. - i.span = i.keyspanIter.Next() - i.checkForwardBound(nil /* prefix */) - i.savedKeyspan() - } else { - // Regardless of the current iterator state, we mark any existing - // span as interleaved when switching to forward iteration, - // justified below. - // - // If the point key is the last key returned: - // pointIter : ... (y) z ... - // keyspanIter : ... ([x - )) ... - // ^ - // The span's start key must be ≤ the point key, otherwise we'd have - // interleaved the span's start key. From a forward-iteration - // perspective, the span's start key is in the past and should be - // considered already-interleaved. - // - // If the span start boundary key is the last key returned: - // pointIter : ... (x) z ... - // keyspanIter : ... ([y - )) ... - // ^ - // i.span.Start is the key we last returned during reverse - // iteration. From the perspective of forward-iteration, its start - // key was just visited. - i.keyspanInterleaved = true + // When switching directions, iterator state corresponding to the + // current iterator position (as indicated by i.pos) is already correct. + // However any state that has yet to be interleaved describes a position + // behind the current iterator position and needs to be updated to + // describe the position ahead of the current iterator position. + switch i.pos { + case posExhausted: + // Nothing to do. The below nextPos call will move both the point + // key and span to their next positions and return + // MIN(point,s.Start). + case posPointKey: + // If we're currently on a point key, the below nextPos will + // correctly Next the point key iterator to the next point key. + // Do we need to move the span forwards? If the current span lies + // entirely behind the current key (!i.withinSpan), then we + // need to move it to the first span in the forward direction. + if !i.withinSpan { + i.saveSpanForward(i.keyspanIter.Next()) + i.savedKeyspan() + } + case posKeyspanStart: + i.withinSpan = true + // Since we're positioned on a Span, the pointIter is positioned + // entirely behind the current iterator position. Reposition it + // ahead of the current iterator position. + i.savePoint(i.pointIter.Next()) + case posKeyspanEnd: + // Since we're positioned on a Span, the pointIter is positioned + // entirely behind of the current iterator position. Reposition it + // ahead the current iterator position. + i.savePoint(i.pointIter.Next()) } + // Fallthrough to calling i.nextPos. } - - // Refresh the point key if the current point key has already been - // interleaved. - if i.pointKeyInterleaved { - i.pointKey, i.pointVal = i.pointIter.Next() - i.pointKeyInterleaved = false - } - // If we already interleaved the current span start key, and the point key - // is ≥ the span's end key, move to the next span. - if i.keyspanInterleaved && i.pointKey != nil && i.span != nil && - i.cmp(i.pointKey.UserKey, i.span.End) >= 0 { - i.span = i.keyspanIter.Next() - i.checkForwardBound(nil /* prefix */) - i.savedKeyspan() - } - return i.interleaveForward(i.lower, nil /* prefix */) + i.nextPos() + return i.yieldPosition(i.lower, i.nextPos) } // NextPrefix implements (base.InternalIterator).NextPrefix. @@ -485,22 +483,24 @@ func (i *InterleavingIter) NextPrefix(succKey []byte) (*base.InternalKey, base.L panic("pebble: cannot switch directions with NextPrefix") } - // Refresh the point key if the current point key has already been - // interleaved, OR if the next uninterleaved point key has the old suffix - // (it's less than succKey). - if i.pointKeyInterleaved || (i.pointKey != nil && i.cmp(i.pointKey.UserKey, succKey) < 0) { - i.pointKey, i.pointVal = i.pointIter.NextPrefix(succKey) - i.pointKeyInterleaved = false + switch i.pos { + case posExhausted: + return nil, base.LazyValue{} + case posPointKey: + i.savePoint(i.pointIter.NextPrefix(succKey)) + if i.withinSpan { + if i.pointKey == nil || i.cmp(i.span.End, i.pointKey.UserKey) <= 0 { + i.pos = posKeyspanEnd + } else { + i.pos = posPointKey + } + } else { + i.computeSmallestPos() + } + case posKeyspanStart, posKeyspanEnd: + i.nextPos() } - // If we already interleaved the current span start key, and the point key - // is ≥ the span's end key, move to the next span. - if i.keyspanInterleaved && i.pointKey != nil && i.span != nil && - i.cmp(i.pointKey.UserKey, i.span.End) >= 0 { - i.span = i.keyspanIter.Next() - i.checkForwardBound(nil) - i.savedKeyspan() - } - return i.interleaveForward(i.lower, nil) + return i.yieldPosition(i.lower, i.nextPos) } // Prev implements (base.InternalIterator).Prev. @@ -516,18 +516,34 @@ func (i *InterleavingIter) Prev() (*base.InternalKey, base.LazyValue) { i.clearMask() } - if i.keyspanInterleaved { - // The current span's start key has already been interleaved in the - // forward direction. The start key may have been interleaved a - // while ago, or it might've been interleaved at the current - // iterator position. If it was interleaved a while ago, the current - // span is still relevant and we should not move the keyspan - // iterator. - // - // If it was just interleaved at the current iterator position, the - // span start was the last key returned to the user. We should - // prev past it so we don't return it again, with an exception. - // Consider span [a, z) and this sequence of iterator calls: + // When switching directions, iterator state corresponding to the + // current iterator position (as indicated by i.pos) is already correct. + // However any state that has yet to be interleaved describes a position + // ahead of the current iterator position and needs to be updated to + // describe the position behind the current iterator position. + switch i.pos { + case posExhausted: + // Nothing to do. The below prevPos call will move both the point + // key and span to previous positions and return MAX(point, s.End). + case posPointKey: + // If we're currently on a point key, the point iterator is in the + // right place and the call to prevPos will correctly Prev the point + // key iterator to the previous point key. Do we need to move the + // span backwards? If the current span lies entirely ahead of the + // current key (!i.withinSpan), then we need to move it to the first + // span in the reverse direction. + if !i.withinSpan { + i.saveSpanBackward(i.keyspanIter.Prev()) + i.savedKeyspan() + } + case posKeyspanStart: + // Since we're positioned on a Span, the pointIter is positioned + // entirely ahead of the current iterator position. Reposition it + // behind the current iterator position. + i.savePoint(i.pointIter.Prev()) + // Without considering truncation of spans to seek keys, the keyspan + // iterator is already in the right place. But consider span [a, z) + // and this sequence of iterator calls: // // SeekGE('c') = c.RANGEKEYSET#72057594037927935 // Prev() = a.RANGEKEYSET#72057594037927935 @@ -535,176 +551,232 @@ func (i *InterleavingIter) Prev() (*base.InternalKey, base.LazyValue) { // If the current span's start key was last surfaced truncated due // to a SeekGE or SeekPrefixGE call, then it's still relevant in the // reverse direction with an untruncated start key. - // - // We can determine whether the last key returned was a point key by - // checking i.pointKeyInterleaved, because every Next/Prev will - // advance the point iterator and reset pointKeyInterleaved if it - // was. - if i.pointKeyInterleaved || i.spanMarkerTruncated { - // The last returned key was a point key, OR a truncated span - // marker key. Don't move, but re-save the span because it - // should no longer be considered truncated or interleaved. - i.savedKeyspan() - } else { - // The last returned key is this key's start boundary, so Prev - // past it so we don't return it again. - i.span = i.keyspanIter.Prev() - i.checkBackwardBound() - i.savedKeyspan() + if i.spanMarkerTruncated { + // When we fallthrough to calling prevPos, we want to move to + // MAX(point, span.Start). We cheat here by claiming we're + // currently on the end boundary, so that we'll move on to the + // untruncated start key if necessary. + i.pos = posKeyspanEnd } - } else { - // If the current span's start key has not been interleaved, then - // the span's start key is greater than the current iterator - // position (denoted in parenthesis), and the current span's start - // key is ahead of our iterator position. Move it to the previous - // span: - // points: (x*) - // span: [y-z)* - i.span = i.keyspanIter.Prev() - i.checkBackwardBound() + case posKeyspanEnd: + // Since we're positioned on a Span, the pointIter is positioned + // entirely ahead of the current iterator position. Reposition it + // behind the current iterator position. + i.savePoint(i.pointIter.Prev()) + } + + if i.spanMarkerTruncated { + // Save the keyspan again to clear truncation. i.savedKeyspan() } - - // The existing point key (denoted below with *) is either the last - // key we returned (the current iterator position): - // points: x (y*) z - // or the upcoming point key in the forward direction if we just - // returned a span start boundary key : - // points: x z* - // spans: ([y-?)) - // direction. Either way, we must move the point iterator backwards. - switch { - case i.pointKey == nil && i.upper == nil: - i.pointKey, i.pointVal = i.pointIter.Last() - case i.pointKey == nil && i.upper != nil: - i.pointKey, i.pointVal = i.pointIter.SeekLT(i.upper, base.SeekLTFlagsNone) - default: - i.pointKey, i.pointVal = i.pointIter.Prev() - } - i.pointKeyInterleaved = false + // Fallthrough to calling i.prevPos. } - - // Refresh the point key if we just returned the current point key. - if i.pointKeyInterleaved { - i.pointKey, i.pointVal = i.pointIter.Prev() - i.pointKeyInterleaved = false - } - // Refresh the span if we just returned the span's start boundary key. - if i.keyspanInterleaved { - i.span = i.keyspanIter.Prev() - i.checkBackwardBound() - i.savedKeyspan() - } - return i.interleaveBackward() + i.prevPos() + return i.yieldPosition(i.lower, i.prevPos) } -func (i *InterleavingIter) interleaveForward( - lowerBound []byte, prefix []byte, -) (*base.InternalKey, base.LazyValue) { - // This loop determines whether a point key or a span marker key should be - // interleaved on each iteration. If masking is disabled and the span is - // nonempty, this loop executes for exactly one iteration. If masking is - // enabled and a masked key is determined to be interleaved next, this loop - // continues until the interleaved key is unmasked. If a span's start key - // should be interleaved next, but the span is empty, the loop continues to - // the next key. - for { - // Check invariants. - if invariants.Enabled { - // INVARIANT: !pointKeyInterleaved - if i.pointKeyInterleaved { - panic("pebble: invariant violation: point key interleaved") - } - switch { - case i.span == nil: - case i.pointKey == nil: - default: - // INVARIANT: !keyspanInterleaved || pointKey < span.End - // The caller is responsible for advancing this span if it's already - // been interleaved and the span ends before the point key. - // Absolute positioning methods will never have already interleaved - // the span's start key, so only Next needs to handle the case where - // pointKey >= span.End. - if i.keyspanInterleaved && i.cmp(i.pointKey.UserKey, i.span.End) >= 0 { - panic("pebble: invariant violation: span interleaved, but point key >= span end") - } - } +// computeSmallestPos sets i.{pos,withinSpan} to: +// +// MIN(i.pointKey, i.span.Start) +func (i *InterleavingIter) computeSmallestPos() { + if i.err == nil { + if i.span != nil && (i.pointKey == nil || i.cmp(i.startKey(), i.pointKey.UserKey) <= 0) { + i.withinSpan = true + i.pos = posKeyspanStart + return } + i.withinSpan = false + if i.pointKey != nil { + i.pos = posPointKey + return + } + } + i.pos = posExhausted +} - // Interleave. +// computeLargestPos sets i.{pos,withinSpan} to: +// +// MAX(i.pointKey, i.span.End) +func (i *InterleavingIter) computeLargestPos() { + if i.err == nil { + if i.span != nil && (i.pointKey == nil || i.cmp(i.span.End, i.pointKey.UserKey) > 0) { + i.withinSpan = true + i.pos = posKeyspanEnd + return + } + i.withinSpan = false + if i.pointKey != nil { + i.pos = posPointKey + return + } + } + i.pos = posExhausted +} + +// nextPos advances the iterator one position in the forward direction. +func (i *InterleavingIter) nextPos() { + if invariants.Enabled { + defer func() { + if i.err != nil && i.pos != posExhausted { + panic(errors.AssertionFailedf("iterator has accumulated error but i.pos = %d", i.pos)) + } + }() + } + // NB: If i.err != nil or any of the positioning methods performed in this + // function result in i.err != nil, we must set i.pos = posExhausted. We + // perform this check explicitly here, but if any of the branches below + // advance either iterator, they must also check i.err and set posExhausted + // if necessary. + if i.err != nil { + i.pos = posExhausted + return + } + + switch i.pos { + case posExhausted: + i.savePoint(i.pointIter.Next()) + i.saveSpanForward(i.keyspanIter.Next()) + i.savedKeyspan() + i.computeSmallestPos() + case posPointKey: + i.savePoint(i.pointIter.Next()) + if i.err != nil { + i.pos = posExhausted + return + } + // If we're not currently within the span, we want to chose the + // MIN(pointKey,span.Start), which is exactly the calculation performed + // by computeSmallestPos. + if !i.withinSpan { + i.computeSmallestPos() + return + } + // i.withinSpan=true + // Since we previously were within the span, we want to choose the + // MIN(pointKey,span.End). switch { case i.span == nil: - // If we're out of spans, just return the point key. - return i.yieldPointKey(false /* covered */) + panic("i.withinSpan=true and i.span=nil") case i.pointKey == nil: - if i.pointKeyInterleaved { - panic("pebble: invariant violation: point key already interleaved") - } - // If we're out of point keys, we need to return a span marker. If - // the current span has already been interleaved, advance it. Since - // there are no more point keys, we don't need to worry about - // advancing past the current point key. - if i.keyspanInterleaved { - i.span = i.keyspanIter.Next() - i.checkForwardBound(prefix) - i.savedKeyspan() - if i.span == nil { - return i.yieldNil() - } - } - if i.span.Empty() { - i.keyspanInterleaved = true - continue - } - return i.yieldSyntheticSpanMarker(lowerBound) + // Since i.withinSpan=true, we step onto the end boundary of the + // keyspan. + i.pos = posKeyspanEnd default: - if i.cmp(i.pointKey.UserKey, i.startKey()) >= 0 { - // The span start key lies before the point key. If we haven't - // interleaved it, we should. - if !i.keyspanInterleaved { - if i.span.Empty() { - if i.pointKey != nil && i.cmp(i.pointKey.UserKey, i.span.End) >= 0 { - // Advance the keyspan iterator, as just flipping - // keyspanInterleaved would likely trip up the invariant check - // above. - i.span = i.keyspanIter.Next() - i.checkForwardBound(prefix) - i.savedKeyspan() - } else { - i.keyspanInterleaved = true - } - continue - } - return i.yieldSyntheticSpanMarker(lowerBound) - } + // i.withinSpan && i.pointKey != nil && i.span != nil + if i.cmp(i.span.End, i.pointKey.UserKey) <= 0 { + i.pos = posKeyspanEnd + } else { + i.pos = posPointKey + } + } + case posKeyspanStart: + // Either a point key or the span's end key comes next. + if i.pointKey != nil && i.cmp(i.pointKey.UserKey, i.span.End) < 0 { + i.pos = posPointKey + } else { + i.pos = posKeyspanEnd + } + case posKeyspanEnd: + i.saveSpanForward(i.keyspanIter.Next()) + i.savedKeyspan() + i.computeSmallestPos() + default: + panic(fmt.Sprintf("unexpected pos=%d", i.pos)) + } +} - // Otherwise, the span's start key is already interleaved and we - // need to return the point key. The current span necessarily - // must cover the point key: - // - // Since the span's start is less than or equal to the point - // key, the only way for this span to not cover the point would - // be if the span's end is less than or equal to the point. - // (For example span = [a, b), point key = c). - // - // However, the invariant at the beginning of the function - // guarantees that if: - // * we have both a point key and a span - // * and the span has already been interleaved - // => then the point key must be less than the span's end, and - // the point key must be covered by the current span. +// prevPos advances the iterator one position in the reverse direction. +func (i *InterleavingIter) prevPos() { + if invariants.Enabled { + defer func() { + if i.err != nil && i.pos != posExhausted { + panic(errors.AssertionFailedf("iterator has accumulated error but i.pos = %d", i.pos)) + } + }() + } + // NB: If i.err != nil or any of the positioning methods performed in this + // function result in i.err != nil, we must set i.pos = posExhausted. We + // perform this check explicitly here, but if any of the branches below + // advance either iterator, they must also check i.err and set posExhausted + // if necessary. + if i.err != nil { + i.pos = posExhausted + return + } - // The span covers the point key. If a SkipPoint hook is - // configured, ask it if we should skip this point key. - // - // But first, we may need to update the mask to the current span - // if we have stepped outside of the span last saved as a mask, - // so that the decision to skip is made with the correct - // knowledge of the covering span. - i.maybeUpdateMask(true /* covered */) + switch i.pos { + case posExhausted: + i.savePoint(i.pointIter.Prev()) + i.saveSpanBackward(i.keyspanIter.Prev()) + i.savedKeyspan() + i.computeLargestPos() + case posPointKey: + i.savePoint(i.pointIter.Prev()) + if i.err != nil { + i.pos = posExhausted + return + } + // If we're not currently covered by the span, we want to chose the + // MAX(pointKey,span.End), which is exactly the calculation performed + // by computeLargestPos. + if !i.withinSpan { + i.computeLargestPos() + return + } + switch { + case i.span == nil: + panic("withinSpan=true, but i.span == nil") + case i.pointKey == nil: + i.pos = posKeyspanEnd + default: + // i.withinSpan && i.pointKey != nil && i.span != nil + if i.cmp(i.span.Start, i.pointKey.UserKey) > 0 { + i.pos = posKeyspanStart + } else { + i.pos = posPointKey + } + } + case posKeyspanStart: + i.saveSpanBackward(i.keyspanIter.Prev()) + i.savedKeyspan() + i.computeLargestPos() + case posKeyspanEnd: + // Either a point key or the span's start key is previous. + if i.pointKey != nil && i.cmp(i.pointKey.UserKey, i.span.Start) >= 0 { + i.pos = posPointKey + } else { + i.pos = posKeyspanStart + } + default: + panic(fmt.Sprintf("unexpected pos=%d", i.pos)) + } +} - if i.mask != nil && i.mask.SkipPoint(i.pointKey.UserKey) { - if i.prefix { +func (i *InterleavingIter) yieldPosition( + lowerBound []byte, advance func(), +) (*base.InternalKey, base.LazyValue) { + // This loop returns the first visible position in the current iteration + // direction. Some positions are not visible and skipped. For example, if + // masking is enabled and the iterator is positioned over a masked point + // key, this loop skips the position. If a span's start key should be + // interleaved next, but the span is empty, the loop continues to the next + // key. Currently, span end keys are also always skipped, and are used only + // for maintaining internal state. + for { + switch i.pos { + case posExhausted: + return i.yieldNil() + case posPointKey: + if i.pointKey == nil { + panic("i.pointKey is nil") + } + + if i.mask != nil { + i.maybeUpdateMask() + if i.withinSpan && i.mask.SkipPoint(i.pointKey.UserKey) { + // The span covers the point key. If a SkipPoint hook is + // configured, ask it if we should skip this point key. + if i.prefix != nil { // During prefix-iteration node, once a point is masked, // all subsequent keys with the same prefix must also be // masked according to the key ordering. We can stop and @@ -721,126 +793,46 @@ func (i *InterleavingIter) interleaveForward( // TODO(jackson): If we thread a base.Comparer through to // InterleavingIter so that we have access to // ImmediateSuccessor, we could use NextPrefix. We'd need to - // tweak the SpanMask interface slightly, but it's probably - // worthwhile. + // tweak the SpanMask interface slightly. - i.pointKey, i.pointVal = i.pointIter.Next() - // We may have just invalidated the invariant that - // ensures the span's End is > the point key, so - // reestablish it before the next iteration. - if i.pointKey != nil && i.cmp(i.pointKey.UserKey, i.span.End) >= 0 { - i.span = i.keyspanIter.Next() - i.checkForwardBound(prefix) - i.savedKeyspan() - } + // Advance beyond the masked point key. + advance() continue } - - // Point key is unmasked but covered. - return i.yieldPointKey(true /* covered */) } - return i.yieldPointKey(false /* covered */) - } - } -} - -func (i *InterleavingIter) interleaveBackward() (*base.InternalKey, base.LazyValue) { - // This loop determines whether a point key or a span's start key should be - // interleaved on each iteration. If masking is disabled and the span is - // nonempty, this loop executes for exactly one iteration. If masking is - // enabled and a masked key is determined to be interleaved next, this loop - // continues until the interleaved key is unmasked. If a span's start key - // should be interleaved next, but the span is empty, the loop continues to - // the next key. - for { - // Check invariants. - if invariants.Enabled { - // INVARIANT: !pointKeyInterleaved - if i.pointKeyInterleaved { - panic("pebble: invariant violation: point key interleaved") - } - } - - // Interleave. - switch { - case i.span == nil: - // If we're out of spans, just return the point key. - return i.yieldPointKey(false /* covered */) - case i.pointKey == nil: - // If we're out of point keys, we need to return a span marker. + return i.yieldPointKey() + case posKeyspanEnd: + // Don't interleave end keys; just advance. + advance() + continue + case posKeyspanStart: + // Don't interleave an empty span. if i.span.Empty() { - i.span = i.keyspanIter.Prev() - i.checkBackwardBound() - i.savedKeyspan() + advance() continue } - return i.yieldSyntheticSpanMarker(i.lower) + return i.yieldSyntheticSpanMarker(lowerBound) default: - // If the span's start key is greater than the point key, return a - // marker for the span. - if i.cmp(i.startKey(), i.pointKey.UserKey) > 0 { - if i.span.Empty() { - i.span = i.keyspanIter.Prev() - i.checkBackwardBound() - i.savedKeyspan() - continue - } - return i.yieldSyntheticSpanMarker(i.lower) - } - // We have a span but it has not been interleaved and begins at a - // key equal to or before the current point key. The point key - // should be interleaved next, if it's not masked. - if i.cmp(i.pointKey.UserKey, i.span.End) < 0 { - // The span covers the point key. The point key might be masked - // too if masking is enabled. - // - // The span may have changed since the last time we updated the - // mask. Consider the following range-key masking scenario: - // - // |--------------) [b,d)@5 - // . c@4 . e@9 - // - // During reverse iteration when we step from e@9 to c@4, we - // enter the span [b,d)@5. Since end boundaries are not - // interleaved, the mask hasn't been updated with the span - // [b,d)@5 yet. We must update the mask before calling - // SkipPoint(c@4) to maintain the SpanMask contract and give the - // mask implementation an opportunity to build the state - // necessary to be able to determine whether [b,d)@5 masks c@4. - i.maybeUpdateMask(true /* covered */) - - // The span covers the point key. If a SkipPoint hook is - // configured, ask it if we should skip this point key. - if i.mask != nil && i.mask.SkipPoint(i.pointKey.UserKey) { - i.pointKey, i.pointVal = i.pointIter.Prev() - continue - } - - // Point key is unmasked but covered. - return i.yieldPointKey(true /* covered */) - } - return i.yieldPointKey(false /* covered */) + panic(fmt.Sprintf("unexpected interleavePos=%d", i.pos)) } } } // keyspanSeekGE seeks the keyspan iterator to the first span covering a key ≥ k. func (i *InterleavingIter) keyspanSeekGE(k []byte, prefix []byte) { - i.span = i.keyspanIter.SeekGE(k) - i.checkForwardBound(prefix) + i.saveSpanForward(i.keyspanIter.SeekGE(k)) i.savedKeyspan() } // keyspanSeekLT seeks the keyspan iterator to the last span covering a key < k. func (i *InterleavingIter) keyspanSeekLT(k []byte) { - i.span = i.keyspanIter.SeekLT(k) - i.checkBackwardBound() + i.saveSpanBackward(i.keyspanIter.SeekLT(k)) // The current span's start key is not guaranteed to be less than key, // because of the bounds enforcement. Consider the following example: // // Bounds are set to [d,e). The user performs a SeekLT(d). The // FragmentIterator.SeekLT lands on a span [b,f). This span has a start key - // less than d, as expected. Above, checkBackwardBound truncates the span to + // less than d, as expected. Above, saveSpanBackward truncates the span to // match the iterator's current bounds, modifying the span to [d,e), which // does not overlap the search space of [-∞, d). // @@ -852,12 +844,21 @@ func (i *InterleavingIter) keyspanSeekLT(k []byte) { i.savedKeyspan() } -func (i *InterleavingIter) checkForwardBound(prefix []byte) { +func (i *InterleavingIter) saveSpanForward(span *Span) { + i.span = span i.truncated = false i.truncatedSpan = Span{} if i.span == nil { + i.err = firstError(i.err, i.keyspanIter.Error()) return } + if invariants.Enabled { + if err := i.keyspanIter.Error(); err != nil { + panic(errors.WithSecondaryError( + errors.AssertionFailedf("pebble: %T keyspan iterator returned non-nil span %s while iter has error", i.keyspanIter, i.span), + err)) + } + } // Check the upper bound if we have one. if i.upper != nil && i.cmp(i.span.Start, i.upper) >= 0 { i.span = nil @@ -886,15 +887,15 @@ func (i *InterleavingIter) checkForwardBound(prefix []byte) { } // If this is a part of a SeekPrefixGE call, we may also need to truncate to // the prefix's bounds. - if prefix != nil { + if i.prefix != nil { if !i.truncated { i.truncated = true i.truncatedSpan = *i.span } - if i.cmp(prefix, i.truncatedSpan.Start) > 0 { - i.truncatedSpan.Start = prefix + if i.cmp(i.prefix, i.truncatedSpan.Start) > 0 { + i.truncatedSpan.Start = i.prefix } - i.nextPrefixBuf = i.comparer.ImmediateSuccessor(i.nextPrefixBuf[:0], prefix) + i.nextPrefixBuf = i.comparer.ImmediateSuccessor(i.nextPrefixBuf[:0], i.prefix) if i.truncated && i.cmp(i.nextPrefixBuf, i.truncatedSpan.End) < 0 { i.truncatedSpan.End = i.nextPrefixBuf } @@ -905,12 +906,22 @@ func (i *InterleavingIter) checkForwardBound(prefix []byte) { } } -func (i *InterleavingIter) checkBackwardBound() { +func (i *InterleavingIter) saveSpanBackward(span *Span) { + i.span = span i.truncated = false i.truncatedSpan = Span{} if i.span == nil { + i.err = firstError(i.err, i.keyspanIter.Error()) return } + if invariants.Enabled { + if err := i.keyspanIter.Error(); err != nil { + panic(errors.WithSecondaryError( + errors.AssertionFailedf("pebble: %T keyspan iterator returned non-nil span %s while iter has error", i.keyspanIter, i.span), + err)) + } + } + // Check the lower bound if we have one. if i.lower != nil && i.cmp(i.span.End, i.lower) <= 0 { i.span = nil @@ -943,15 +954,12 @@ func (i *InterleavingIter) checkBackwardBound() { } func (i *InterleavingIter) yieldNil() (*base.InternalKey, base.LazyValue) { - i.spanCoversKey = false + i.withinSpan = false i.clearMask() return i.verify(nil, base.LazyValue{}) } -func (i *InterleavingIter) yieldPointKey(covered bool) (*base.InternalKey, base.LazyValue) { - i.pointKeyInterleaved = true - i.spanCoversKey = covered - i.maybeUpdateMask(covered) +func (i *InterleavingIter) yieldPointKey() (*base.InternalKey, base.LazyValue) { return i.verify(i.pointKey, i.pointVal) } @@ -960,8 +968,6 @@ func (i *InterleavingIter) yieldSyntheticSpanMarker( ) (*base.InternalKey, base.LazyValue) { i.spanMarker.UserKey = i.startKey() i.spanMarker.Trailer = base.MakeTrailer(base.InternalKeySeqNumMax, i.span.Keys[0].Kind()) - i.keyspanInterleaved = true - i.spanCoversKey = true // Truncate the key we return to our lower bound if we have one. Note that // we use the lowerBound function parameter, not i.lower. The lowerBound @@ -992,13 +998,13 @@ func (i *InterleavingIter) yieldSyntheticSpanMarker( i.spanMarker.UserKey = i.keyBuf i.spanMarkerTruncated = true } - i.maybeUpdateMask(true /* covered */) + i.maybeUpdateMask() return i.verify(&i.spanMarker, base.LazyValue{}) } func (i *InterleavingIter) disablePrefixMode() { - if i.prefix { - i.prefix = false + if i.prefix != nil { + i.prefix = nil // Clear the existing span. It may not hold the true end bound of the // underlying span. i.span = nil @@ -1012,52 +1018,42 @@ func (i *InterleavingIter) verify( // production builds elide this entire function. if invariants.Enabled { switch { - case k != nil && !i.keyspanInterleaved && !i.pointKeyInterleaved: - panic("pebble: invariant violation: both keys marked as noninterleaved") - case i.dir == -1 && k != nil && i.keyspanInterleaved == i.pointKeyInterleaved: - // During reverse iteration, if we're returning a key, either the span's - // start key must have been interleaved OR the current point key value - // is being returned, not both. - // - // This invariant holds because in reverse iteration the start key of the - // span behaves like a point. Once the start key is interleaved, we move - // the keyspan iterator to the previous span. - panic(fmt.Sprintf("pebble: invariant violation: interleaving (point %t, span %t)", - i.pointKeyInterleaved, i.keyspanInterleaved)) case i.dir == -1 && i.spanMarkerTruncated: panic("pebble: invariant violation: truncated span key in reverse iteration") case k != nil && i.lower != nil && i.cmp(k.UserKey, i.lower) < 0: panic("pebble: invariant violation: key < lower bound") case k != nil && i.upper != nil && i.cmp(k.UserKey, i.upper) >= 0: panic("pebble: invariant violation: key ≥ upper bound") - case i.span != nil && k != nil && i.mask != nil && i.pointKeyInterleaved && - i.cmp(k.UserKey, i.span.Start) >= 0 && i.cmp(k.UserKey, i.span.End) < 0 && i.mask.SkipPoint(k.UserKey): - panic("pebble: invariant violation: point key eligible for skipping returned") + case i.err != nil && k != nil: + panic("pebble: invariant violation: accumulated error swallowed") + case i.err == nil && i.pointIter.Error() != nil: + panic("pebble: invariant violation: pointIter swallowed") + case i.err == nil && i.keyspanIter.Error() != nil: + panic("pebble: invariant violation: keyspanIter error swallowed") } } return k, v } func (i *InterleavingIter) savedKeyspan() { - i.keyspanInterleaved = false i.spanMarkerTruncated = false i.maskSpanChangedCalled = false } -// maybeUpdateMask updates the current mask, if a mask is configured and -// the mask hasn't been updated with the current keyspan yet. -func (i *InterleavingIter) maybeUpdateMask(covered bool) { - if i.mask != nil { - if !covered || i.span.Empty() { - i.clearMask() - } else if !i.maskSpanChangedCalled { - if i.truncated { - i.mask.SpanChanged(&i.truncatedSpan) - } else { - i.mask.SpanChanged(i.span) - } - i.maskSpanChangedCalled = true - } +// updateMask updates the current mask, if a mask is configured and the mask +// hasn't been updated with the current keyspan yet. +func (i *InterleavingIter) maybeUpdateMask() { + switch { + case i.mask == nil, i.maskSpanChangedCalled: + return + case !i.withinSpan || i.span.Empty(): + i.clearMask() + case i.truncated: + i.mask.SpanChanged(&i.truncatedSpan) + i.maskSpanChangedCalled = true + default: + i.mask.SpanChanged(i.span) + i.maskSpanChangedCalled = true } } @@ -1077,6 +1073,20 @@ func (i *InterleavingIter) startKey() []byte { return i.span.Start } +func (i *InterleavingIter) savePoint(key *base.InternalKey, value base.LazyValue) { + i.pointKey, i.pointVal = key, value + if key == nil { + i.err = firstError(i.err, i.pointIter.Error()) + } + if invariants.Enabled { + if err := i.pointIter.Error(); key != nil && err != nil { + panic(errors.WithSecondaryError( + errors.AssertionFailedf("pebble: %T point iterator returned non-nil key %q while iter has error", i.pointIter, key), + err)) + } + } +} + // Span returns the span covering the last key returned, if any. A span key is // considered to 'cover' a key if the key falls within the span's user key // bounds. The returned span is owned by the InterleavingIter. The caller is @@ -1084,13 +1094,12 @@ func (i *InterleavingIter) startKey() []byte { // // Span will never return an invalid or empty span. func (i *InterleavingIter) Span() *Span { - if !i.spanCoversKey || i.span.Empty() { + if !i.withinSpan || len(i.span.Keys) == 0 { return nil } else if i.truncated { return &i.truncatedSpan - } else { - return i.span } + return i.span } // SetBounds implements (base.InternalIterator).SetBounds. @@ -1111,7 +1120,7 @@ func (i *InterleavingIter) Invalidate() { // Error implements (base.InternalIterator).Error. func (i *InterleavingIter) Error() error { - return firstError(i.pointIter.Error(), i.keyspanIter.Error()) + return i.err } // Close implements (base.InternalIterator).Close. diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go index 4798561..7f8ceb8 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go @@ -51,6 +51,8 @@ type FragmentIterator interface { Prev() *Span // Error returns any accumulated error. + // + // TODO(jackson): Lift errors into return values on the positioning methods. Error() error // Close closes the iterator and returns any accumulated error. Exhausting @@ -62,7 +64,7 @@ type FragmentIterator interface { // TableNewSpanIter creates a new iterator for range key spans for the given // file. -type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions *SpanIterOptions) (FragmentIterator, error) +type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions SpanIterOptions) (FragmentIterator, error) // SpanIterOptions is a subset of IterOptions that are necessary to instantiate // per-sstable span iterators. diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go index 047c6a2..6dd7ac6 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go @@ -98,7 +98,7 @@ func (l *LevelIter) Init( ) { l.err = nil l.level = level - l.tableOpts.RangeKeyFilters = opts.RangeKeyFilters + l.tableOpts = opts l.cmp = cmp l.iterFile = nil l.newIter = newIter @@ -179,7 +179,7 @@ func (l *LevelIter) loadFile(file *manifest.FileMetadata, dir int) loadFileRetur return noFileLoaded } if indicator != fileAlreadyLoaded { - l.iter, l.err = l.newIter(file, &l.tableOpts) + l.iter, l.err = l.newIter(file, l.tableOpts) indicator = newFileLoaded } if l.err != nil { diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go index 2476024..c73ba59 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go @@ -25,49 +25,6 @@ import ( // MergingIter implementation, but will require a bit of plumbing to thread the // Equal function. -// Transformer defines a transformation to be applied to a Span. -type Transformer interface { - // Transform takes a Span as input and writes the transformed Span to the - // provided output *Span pointer. The output Span's Keys slice may be reused - // by Transform to reduce allocations. - Transform(cmp base.Compare, in Span, out *Span) error -} - -// The TransformerFunc type is an adapter to allow the use of ordinary functions -// as Transformers. If f is a function with the appropriate signature, -// TransformerFunc(f) is a Transformer that calls f. -type TransformerFunc func(base.Compare, Span, *Span) error - -// Transform calls f(cmp, in, out). -func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error { - return tf(cmp, in, out) -} - -var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error { - dst.Start, dst.End = s.Start, s.End - dst.Keys = append(dst.Keys[:0], s.Keys...) - return nil -}) - -// VisibleTransform filters keys that are invisible at the provided snapshot -// sequence number. -func VisibleTransform(snapshot uint64) Transformer { - return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error { - dst.Start, dst.End = s.Start, s.End - dst.Keys = dst.Keys[:0] - for _, k := range s.Keys { - // NB: The InternalKeySeqNumMax value is used for the batch snapshot - // because a batch's visible span keys are filtered when they're - // fragmented. There's no requirement to enforce visibility at - // iteration time. - if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) { - dst.Keys = append(dst.Keys, k) - } - } - return nil - }) -} - // MergingIter merges spans across levels of the LSM, exposing an iterator over // spans that yields sets of spans fragmented at unique user key boundaries. // diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go new file mode 100644 index 0000000..b5e8735 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/transformer.go @@ -0,0 +1,50 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package keyspan + +import "github.com/cockroachdb/pebble/internal/base" + +// Transformer defines a transformation to be applied to a Span. +type Transformer interface { + // Transform takes a Span as input and writes the transformed Span to the + // provided output *Span pointer. The output Span's Keys slice may be reused + // by Transform to reduce allocations. + Transform(cmp base.Compare, in Span, out *Span) error +} + +// The TransformerFunc type is an adapter to allow the use of ordinary functions +// as Transformers. If f is a function with the appropriate signature, +// TransformerFunc(f) is a Transformer that calls f. +type TransformerFunc func(base.Compare, Span, *Span) error + +// Transform calls f(cmp, in, out). +func (tf TransformerFunc) Transform(cmp base.Compare, in Span, out *Span) error { + return tf(cmp, in, out) +} + +var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst *Span) error { + dst.Start, dst.End = s.Start, s.End + dst.Keys = append(dst.Keys[:0], s.Keys...) + return nil +}) + +// VisibleTransform filters keys that are invisible at the provided snapshot +// sequence number. +func VisibleTransform(snapshot uint64) Transformer { + return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error { + dst.Start, dst.End = s.Start, s.End + dst.Keys = dst.Keys[:0] + for _, k := range s.Keys { + // NB: The InternalKeySeqNumMax value is used for the batch snapshot + // because a batch's visible span keys are filtered when they're + // fragmented. There's no requirement to enforce visibility at + // iteration time. + if base.Visible(k.SeqNum(), snapshot, base.InternalKeySeqNumMax) { + dst.Keys = append(dst.Keys, k) + } + } + return nil + }) +} diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go index af3dddd..c0e609b 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go @@ -10,13 +10,18 @@ import "github.com/cockroachdb/pebble/internal/base" // truncated to be contained within the range [lower, upper). If start and end // are specified, filter out any spans that are completely outside those bounds. func Truncate( - cmp base.Compare, iter FragmentIterator, lower, upper []byte, start, end *base.InternalKey, + cmp base.Compare, + iter FragmentIterator, + lower, upper []byte, + start, end *base.InternalKey, + panicOnUpperTruncate bool, ) FragmentIterator { return Filter(iter, func(in *Span, out *Span) (keep bool) { out.Start, out.End = in.Start, in.End out.Keys = append(out.Keys[:0], in.Keys...) - // Ignore this span if it lies completely outside [start, end]. + // Ignore this span if it lies completely outside start, end. Note that + // end endInclusive indicated whether end is inclusive. // // The comparison between s.End and start is by user key only, as // the span is exclusive at s.End, so comparing by user keys @@ -48,13 +53,21 @@ func Truncate( // Wholly within the end bound. Keep it. } } + + var truncated bool // Truncate the bounds to lower and upper. if cmp(in.Start, lower) < 0 { out.Start = lower } if cmp(in.End, upper) > 0 { + truncated = true out.End = upper } + + if panicOnUpperTruncate && truncated { + panic("pebble: upper bound should not be truncated") + } + return !out.Empty() && cmp(out.Start, out.End) < 0 - }) + }, cmp) } diff --git a/vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go b/vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go index 66464c9..6e56af8 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go +++ b/vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go @@ -108,7 +108,7 @@ type annotation struct { } type leafNode struct { - ref int32 + ref atomic.Int32 count int16 leaf bool // subtreeCount holds the count of files in the entire subtree formed by @@ -139,13 +139,13 @@ func leafToNode(ln *leafNode) *node { func newLeafNode() *node { n := leafToNode(new(leafNode)) n.leaf = true - n.ref = 1 + n.ref.Store(1) return n } func newNode() *node { n := new(node) - n.ref = 1 + n.ref.Store(1) return n } @@ -159,7 +159,7 @@ func newNode() *node { // When a node is cloned, the provided pointer will be redirected to the new // mutable node. func mut(n **node) *node { - if atomic.LoadInt32(&(*n).ref) == 1 { + if (*n).ref.Load() == 1 { // Exclusive ownership. Can mutate in place. // Whenever a node will be mutated, reset its annotations to be marked @@ -186,7 +186,7 @@ func mut(n **node) *node { // incRef acquires a reference to the node. func (n *node) incRef() { - atomic.AddInt32(&n.ref, 1) + n.ref.Add(1) } // decRef releases a reference to the node. If requested, the method will unref @@ -195,8 +195,8 @@ func (n *node) incRef() { // new nodes pass contentsToo=false to preserve existing reference counts during // operations that should yield a net-zero change to descendant refcounts. // When a node is released, its contained files are dereferenced. -func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) { - if atomic.AddInt32(&n.ref, -1) > 0 { +func (n *node) decRef(contentsToo bool, obsolete *[]*FileBacking) { + if n.ref.Add(-1) > 0 { // Other references remain. Can't free. return } @@ -207,7 +207,7 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) { // nodes, and they want to preserve the existing reference count. if contentsToo { for _, f := range n.items[:n.count] { - if atomic.AddInt32(&f.Refs, -1) == 0 { + if f.Unref() == 0 { // There are two sources of node dereferences: tree mutations // and Version dereferences. Files should only be made obsolete // during Version dereferences, during which `obsolete` will be @@ -215,7 +215,13 @@ func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) { if obsolete == nil { panic(fmt.Sprintf("file metadata %s dereferenced to zero during tree mutation", f.FileNum)) } - *obsolete = append(*obsolete, f) + // Reference counting is performed on the FileBacking. In the case + // of a virtual sstable, this reference counting is performed on + // a FileBacking which is shared by every single virtual sstable + // with the same backing sstable. If the reference count hits 0, + // then we know that the FileBacking won't be required by any + // sstable in Pebble, and that the backing sstable can be deleted. + *obsolete = append(*obsolete, f.FileBacking) } } if !n.leaf { @@ -241,7 +247,7 @@ func (n *node) clone() *node { c.subtreeCount = n.subtreeCount // Increase the refcount of each contained item. for _, f := range n.items[:n.count] { - atomic.AddInt32(&f.Refs, 1) + f.Ref() } if !c.leaf { // Copy children and increase each refcount. @@ -761,8 +767,8 @@ type btree struct { // Release dereferences and clears the root node of the btree, removing all // items from the btree. In doing so, it decrements contained file counts. -// It returns a slice of newly obsolete files, if any. -func (t *btree) Release() (obsolete []*FileMetadata) { +// It returns a slice of newly obsolete backing files, if any. +func (t *btree) Release() (obsolete []*FileBacking) { if t.root != nil { t.root.decRef(true /* contentsToo */, &obsolete) t.root = nil @@ -800,7 +806,7 @@ func (t *btree) Delete(item *FileMetadata) (obsolete bool) { return false } if out := mut(&t.root).Remove(t.cmp, item); out != nil { - obsolete = atomic.AddInt32(&out.Refs, -1) == 0 + obsolete = out.Unref() == 0 } if invariants.Enabled { t.root.verifyInvariants() @@ -832,7 +838,7 @@ func (t *btree) Insert(item *FileMetadata) error { newRoot.subtreeCount = t.root.subtreeCount + splitNode.subtreeCount + 1 t.root = newRoot } - atomic.AddInt32(&item.Refs, 1) + item.Ref() err := mut(&t.root).Insert(t.cmp, item) if invariants.Enabled { t.root.verifyInvariants() diff --git a/vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go b/vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go index 3e1d26d..d48e277 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go +++ b/vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go @@ -15,20 +15,28 @@ import ( // LevelMetadata contains metadata for all of the files within // a level of the LSM. type LevelMetadata struct { - level int - tree btree + level int + totalSize uint64 + // NumVirtual is the number of virtual sstables in the level. + NumVirtual uint64 + // VirtualSize is the size of the virtual sstables in the level. + VirtualSize uint64 + tree btree } // clone makes a copy of the level metadata, implicitly increasing the ref // count of every file contained within lm. func (lm *LevelMetadata) clone() LevelMetadata { return LevelMetadata{ - level: lm.level, - tree: lm.tree.Clone(), + level: lm.level, + totalSize: lm.totalSize, + NumVirtual: lm.NumVirtual, + VirtualSize: lm.VirtualSize, + tree: lm.tree.Clone(), } } -func (lm *LevelMetadata) release() (obsolete []*FileMetadata) { +func (lm *LevelMetadata) release() (obsolete []*FileBacking) { return lm.tree.Release() } @@ -40,6 +48,13 @@ func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetad var lm LevelMetadata lm.level = level lm.tree, _ = makeBTree(bcmp, files) + for _, f := range files { + lm.totalSize += f.Size + if f.Virtual { + lm.NumVirtual++ + lm.VirtualSize += f.Size + } + } return lm } @@ -52,6 +67,27 @@ func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) { return t, newLevelSlice(t.Iter()) } +func (lm *LevelMetadata) insert(f *FileMetadata) error { + if err := lm.tree.Insert(f); err != nil { + return err + } + lm.totalSize += f.Size + if f.Virtual { + lm.NumVirtual++ + lm.VirtualSize += f.Size + } + return nil +} + +func (lm *LevelMetadata) remove(f *FileMetadata) bool { + lm.totalSize -= f.Size + if f.Virtual { + lm.NumVirtual-- + lm.VirtualSize -= f.Size + } + return lm.tree.Delete(f) +} + // Empty indicates whether there are any files in the level. func (lm *LevelMetadata) Empty() bool { return lm.tree.Count() == 0 @@ -62,6 +98,11 @@ func (lm *LevelMetadata) Len() int { return lm.tree.Count() } +// Size returns the cumulative size of all the files within the level. +func (lm *LevelMetadata) Size() uint64 { + return lm.totalSize +} + // Iter constructs a LevelIterator over the entire level. func (lm *LevelMetadata) Iter() LevelIterator { return LevelIterator{iter: lm.tree.Iter()} @@ -280,6 +321,32 @@ func (ls *LevelSlice) SizeSum() uint64 { return sum } +// NumVirtual returns the number of virtual sstables in the level. Its runtime is +// linear in the length of the slice. +func (ls *LevelSlice) NumVirtual() uint64 { + var n uint64 + iter := ls.Iter() + for f := iter.First(); f != nil; f = iter.Next() { + if f.Virtual { + n++ + } + } + return n +} + +// VirtualSizeSum returns the sum of the sizes of the virtual sstables in the +// level. +func (ls *LevelSlice) VirtualSizeSum() uint64 { + var sum uint64 + iter := ls.Iter() + for f := iter.First(); f != nil; f = iter.Next() { + if f.Virtual { + sum += f.Size + } + } + return sum +} + // Reslice constructs a new slice backed by the same underlying level, with // new start and end positions. Reslice invokes the provided function, passing // two LevelIterators: one positioned to i's inclusive start and one diff --git a/vendor/github.com/cockroachdb/pebble/internal/manifest/version.go b/vendor/github.com/cockroachdb/pebble/internal/manifest/version.go index 313fc67..0a3b774 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/manifest/version.go +++ b/vendor/github.com/cockroachdb/pebble/internal/manifest/version.go @@ -49,6 +49,9 @@ type TableStats struct { // The number of point and range deletion entries in the table. NumDeletions uint64 // NumRangeKeySets is the total number of range key sets in the table. + // + // NB: If there's a chance that the sstable contains any range key sets, + // then NumRangeKeySets must be > 0. NumRangeKeySets uint64 // Estimate of the total disk space that may be dropped by this table's // point deletions by compacting them. @@ -60,9 +63,11 @@ type TableStats struct { // overlapping data in L0 and ignores L0 sublevels, but the error that // introduces is expected to be small. // - // Tables in the bottommost level of the LSM may have a nonzero estimate - // if snapshots or move compactions prevented the elision of their range - // tombstones. + // Tables in the bottommost level of the LSM may have a nonzero estimate if + // snapshots or move compactions prevented the elision of their range + // tombstones. A table in the bottommost level that was ingested into L6 + // will have a zero estimate, because the file's sequence numbers indicate + // that the tombstone cannot drop any data contained within the file itself. RangeDeletionsBytesEstimate uint64 // Total size of value blocks and value index block. ValueBlocksSize uint64 @@ -119,43 +124,72 @@ func (s CompactionState) String() string { } } -// FileMetadata holds the metadata for an on-disk table. +// FileMetadata is maintained for leveled-ssts, i.e., they belong to a level of +// some version. FileMetadata does not contain the actual level of the sst, +// since such leveled-ssts can move across levels in different versions, while +// sharing the same FileMetadata. There are two kinds of leveled-ssts, physical +// and virtual. Underlying both leveled-ssts is a backing-sst, for which the +// only state is FileBacking. A backing-sst is level-less. It is possible for a +// backing-sst to be referred to by a physical sst in one version and by one or +// more virtual ssts in one or more versions. A backing-sst becomes obsolete +// and can be deleted once it is no longer required by any physical or virtual +// sst in any version. +// +// We maintain some invariants: +// +// 1. Each physical and virtual sst will have a unique FileMetadata.FileNum, +// and there will be exactly one FileMetadata associated with the FileNum. +// +// 2. Within a version, a backing-sst is either only referred to by one +// physical sst or one or more virtual ssts. +// +// 3. Once a backing-sst is referred to by a virtual sst in the latest version, +// it cannot go back to being referred to by a physical sst in any future +// version. +// +// Once a physical sst is no longer needed by any version, we will no longer +// maintain the file metadata associated with it. We will still maintain the +// FileBacking associated with the physical sst if the backing sst is required +// by any virtual ssts in any version. type FileMetadata struct { - // Atomic contains fields which are accessed atomically. Go allocations - // are guaranteed to be 64-bit aligned which we take advantage of by - // placing the 64-bit fields which we access atomically at the beginning - // of the FileMetadata struct. For more information, see - // https://golang.org/pkg/sync/atomic/#pkg-note-BUG. - Atomic struct { - // AllowedSeeks is used to determine if a file should be picked for - // a read triggered compaction. It is decremented when read sampling - // in pebble.Iterator after every after every positioning operation - // that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc). - AllowedSeeks int64 + // AllowedSeeks is used to determine if a file should be picked for + // a read triggered compaction. It is decremented when read sampling + // in pebble.Iterator after every after every positioning operation + // that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc). + AllowedSeeks atomic.Int64 - // statsValid is 1 if stats have been loaded for the table. The - // TableStats structure is populated only if valid is 1. - statsValid uint32 - } + // statsValid indicates if stats have been loaded for the table. The + // TableStats structure is populated only if valid is true. + statsValid atomic.Bool + + // FileBacking is the state which backs either a physical or virtual + // sstables. + FileBacking *FileBacking // InitAllowedSeeks is the inital value of allowed seeks. This is used // to re-set allowed seeks on a file once it hits 0. InitAllowedSeeks int64 - - // Reference count for the file: incremented when a file is added to a - // version and decremented when the version is unreferenced. The file is - // obsolete when the reference count falls to zero. - Refs int32 // FileNum is the file number. + // + // INVARIANT: when !FileMetadata.Virtual, FileNum == FileBacking.DiskFileNum. FileNum base.FileNum - // Size is the size of the file, in bytes. + // Size is the size of the file, in bytes. Size is an approximate value for + // virtual sstables. + // + // INVARIANTS: + // - When !FileMetadata.Virtual, Size == FileBacking.Size. + // - Size should be non-zero. Size 0 virtual sstables must not be created. Size uint64 // File creation time in seconds since the epoch (1970-01-01 00:00:00 // UTC). For ingested sstables, this corresponds to the time the file was - // ingested. + // ingested. For virtual sstables, this corresponds to the wall clock time + // when the FileMetadata for the virtual sstable was first created. CreationTime int64 - // Smallest and largest sequence numbers in the table, across both point and - // range keys. + // Lower and upper bounds for the smallest and largest sequence numbers in + // the table, across both point and range keys. For physical sstables, these + // values are tight bounds. For virtual sstables, there is no guarantee that + // there will be keys with SmallestSeqNum or LargestSeqNum within virtual + // sstable bounds. SmallestSeqNum uint64 LargestSeqNum uint64 // SmallestPointKey and LargestPointKey are the inclusive bounds for the @@ -178,8 +212,17 @@ type FileMetadata struct { Smallest InternalKey Largest InternalKey // Stats describe table statistics. Protected by DB.mu. + // + // For virtual sstables, set stats upon virtual sstable creation as + // asynchronous computation of stats is not currently supported. + // + // TODO(bananabrick): To support manifest replay for virtual sstables, we + // probably need to compute virtual sstable stats asynchronously. Otherwise, + // we'd have to write virtual sstable stats to the version edit. Stats TableStats + // For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and + // pick L0 compactions. Only accurate for the most recent Version. SubLevel int L0Index int minIntervalIndex int @@ -188,9 +231,6 @@ type FileMetadata struct { // NB: the alignment of this struct is 8 bytes. We pack all the bools to // ensure an optimal packing. - // For L0 files only. Protected by DB.mu. Used to generate L0 sublevels and - // pick L0 compactions. Only accurate for the most recent Version. - // // IsIntraL0Compacting is set to True if this file is part of an intra-L0 // compaction. When it's true, IsCompacting must also return true. If // Compacting is true and IsIntraL0Compacting is false for an L0 file, the @@ -226,6 +266,212 @@ type FileMetadata struct { // key type (point or range) corresponds to the smallest and largest overall // table bounds. boundTypeSmallest, boundTypeLargest boundType + // Virtual is true if the FileMetadata belongs to a virtual sstable. + Virtual bool +} + +// PhysicalFileMeta is used by functions which want a guarantee that their input +// belongs to a physical sst and not a virtual sst. +// +// NB: This type should only be constructed by calling +// FileMetadata.PhysicalMeta. +type PhysicalFileMeta struct { + *FileMetadata +} + +// VirtualFileMeta is used by functions which want a guarantee that their input +// belongs to a virtual sst and not a physical sst. +// +// A VirtualFileMeta inherits all the same fields as a FileMetadata. These +// fields have additional invariants imposed on them, and/or slightly varying +// meanings: +// - Smallest and Largest (and their counterparts +// {Smallest, Largest}{Point,Range}Key) remain tight bounds that represent a +// key at that exact bound. We make the effort to determine the next smallest +// or largest key in an sstable after virtualizing it, to maintain this +// tightness. If the largest is a sentinel key (IsExclusiveSentinel()), it +// could mean that a rangedel or range key ends at that user key, or has been +// truncated to that user key. +// - One invariant is that if a rangedel or range key is truncated on its +// upper bound, the virtual sstable *must* have a rangedel or range key +// sentinel key as its upper bound. This is because truncation yields +// an exclusive upper bound for the rangedel/rangekey, and if there are +// any points at that exclusive upper bound within the same virtual +// sstable, those could get uncovered by this truncation. We enforce this +// invariant in calls to keyspan.Truncate. +// - Size is an estimate of the size of the virtualized portion of this sstable. +// The underlying file's size is stored in FileBacking.Size, though it could +// also be estimated or could correspond to just the referenced portion of +// a file (eg. if the file originated on another node). +// - Size must be > 0. +// - SmallestSeqNum and LargestSeqNum are loose bounds for virtual sstables. +// This means that all keys in the virtual sstable must have seqnums within +// [SmallestSeqNum, LargestSeqNum], however there's no guarantee that there's +// a key with a seqnum at either of the bounds. Calculating tight seqnum +// bounds would be too expensive and deliver little value. +// +// NB: This type should only be constructed by calling FileMetadata.VirtualMeta. +type VirtualFileMeta struct { + *FileMetadata +} + +// PhysicalMeta should be the only source of creating the PhysicalFileMeta +// wrapper type. +func (m *FileMetadata) PhysicalMeta() PhysicalFileMeta { + if m.Virtual { + panic("pebble: file metadata does not belong to a physical sstable") + } + return PhysicalFileMeta{ + m, + } +} + +// VirtualMeta should be the only source of creating the VirtualFileMeta wrapper +// type. +func (m *FileMetadata) VirtualMeta() VirtualFileMeta { + if !m.Virtual { + panic("pebble: file metadata does not belong to a virtual sstable") + } + return VirtualFileMeta{ + m, + } +} + +// FileBacking either backs a single physical sstable, or one or more virtual +// sstables. +// +// See the comment above the FileMetadata type for sstable terminology. +type FileBacking struct { + // Reference count for the backing file on disk: incremented when a + // physical or virtual sstable which is backed by the FileBacking is + // added to a version and decremented when the version is unreferenced. + // We ref count in order to determine when it is safe to delete a + // backing sst file from disk. The backing file is obsolete when the + // reference count falls to zero. + refs atomic.Int32 + // latestVersionRefs are the references to the FileBacking in the + // latest version. This reference can be through a single physical + // sstable in the latest version, or one or more virtual sstables in the + // latest version. + // + // INVARIANT: latestVersionRefs <= refs. + latestVersionRefs atomic.Int32 + // VirtualizedSize is set iff the backing sst is only referred to by + // virtual ssts in the latest version. VirtualizedSize is the sum of the + // virtual sstable sizes of all of the virtual sstables in the latest + // version which are backed by the physical sstable. When a virtual + // sstable is removed from the latest version, we will decrement the + // VirtualizedSize. During compaction picking, we'll compensate a + // virtual sstable file size by + // (FileBacking.Size - FileBacking.VirtualizedSize) / latestVersionRefs. + // The intuition is that if FileBacking.Size - FileBacking.VirtualizedSize + // is high, then the space amplification due to virtual sstables is + // high, and we should pick the virtual sstable with a higher priority. + // + // TODO(bananabrick): Compensate the virtual sstable file size using + // the VirtualizedSize during compaction picking and test. + VirtualizedSize atomic.Uint64 + DiskFileNum base.DiskFileNum + Size uint64 +} + +// InitPhysicalBacking allocates and sets the FileBacking which is required by a +// physical sstable FileMetadata. +// +// Ensure that the state required by FileBacking, such as the FileNum, is +// already set on the FileMetadata before InitPhysicalBacking is called. +// Calling InitPhysicalBacking only after the relevant state has been set in the +// FileMetadata is not necessary in tests which don't rely on FileBacking. +func (m *FileMetadata) InitPhysicalBacking() { + if m.Virtual { + panic("pebble: virtual sstables should use a pre-existing FileBacking") + } + if m.FileBacking == nil { + m.FileBacking = &FileBacking{Size: m.Size, DiskFileNum: m.FileNum.DiskFileNum()} + } +} + +// InitProviderBacking creates a new FileBacking for a file backed by +// an objstorage.Provider. +func (m *FileMetadata) InitProviderBacking(fileNum base.DiskFileNum) { + if !m.Virtual { + panic("pebble: provider-backed sstables must be virtual") + } + if m.FileBacking == nil { + m.FileBacking = &FileBacking{DiskFileNum: fileNum} + } +} + +// ValidateVirtual should be called once the FileMetadata for a virtual sstable +// is created to verify that the fields of the virtual sstable are sound. +func (m *FileMetadata) ValidateVirtual(createdFrom *FileMetadata) { + if !m.Virtual { + panic("pebble: invalid virtual sstable") + } + + if createdFrom.SmallestSeqNum != m.SmallestSeqNum { + panic("pebble: invalid smallest sequence number for virtual sstable") + } + + if createdFrom.LargestSeqNum != m.LargestSeqNum { + panic("pebble: invalid largest sequence number for virtual sstable") + } + + if createdFrom.FileBacking != nil && createdFrom.FileBacking != m.FileBacking { + panic("pebble: invalid physical sstable state for virtual sstable") + } + + if m.Size == 0 { + panic("pebble: virtual sstable size must be set upon creation") + } +} + +// Refs returns the refcount of backing sstable. +func (m *FileMetadata) Refs() int32 { + return m.FileBacking.refs.Load() +} + +// Ref increments the ref count associated with the backing sstable. +func (m *FileMetadata) Ref() { + m.FileBacking.refs.Add(1) +} + +// Unref decrements the ref count associated with the backing sstable. +func (m *FileMetadata) Unref() int32 { + v := m.FileBacking.refs.Add(-1) + if invariants.Enabled && v < 0 { + panic("pebble: invalid FileMetadata refcounting") + } + return v +} + +// LatestRef increments the latest ref count associated with the backing +// sstable. +func (m *FileMetadata) LatestRef() { + m.FileBacking.latestVersionRefs.Add(1) + + if m.Virtual { + m.FileBacking.VirtualizedSize.Add(m.Size) + } +} + +// LatestUnref decrements the latest ref count associated with the backing +// sstable. +func (m *FileMetadata) LatestUnref() int32 { + if m.Virtual { + m.FileBacking.VirtualizedSize.Add(-m.Size) + } + + v := m.FileBacking.latestVersionRefs.Add(-1) + if invariants.Enabled && v < 0 { + panic("pebble: invalid FileMetadata latest refcounting") + } + return v +} + +// LatestRefs returns the latest ref count associated with the backing sstable. +func (m *FileMetadata) LatestRefs() int32 { + return m.FileBacking.latestVersionRefs.Load() } // SetCompactionState transitions this file's compaction state to the given @@ -263,22 +509,14 @@ func (m *FileMetadata) IsCompacting() bool { // returns true, the Stats field may be read (with or without holding the // database mutex). func (m *FileMetadata) StatsValid() bool { - return atomic.LoadUint32(&m.Atomic.statsValid) == 1 -} - -// StatsValidLocked returns true if the table stats have been populated. -// StatsValidLocked requires DB.mu is held when it's invoked, and it avoids the -// overhead of an atomic load. This is possible because table stats validity is -// only set while DB.mu is held. -func (m *FileMetadata) StatsValidLocked() bool { - return m.Atomic.statsValid == 1 + return m.statsValid.Load() } // StatsMarkValid marks the TableStats as valid. The caller must hold DB.mu // while populating TableStats and calling StatsMarkValud. Once stats are // populated, they must not be mutated. func (m *FileMetadata) StatsMarkValid() { - atomic.StoreUint32(&m.Atomic.statsValid, 1) + m.statsValid.Store(true) } // ExtendPointKeyBounds attempts to extend the lower and upper point key bounds @@ -369,6 +607,13 @@ func (m *FileMetadata) Overlaps(cmp Compare, start []byte, end []byte, exclusive return true } +// ContainedWithinSpan returns true if the file key range completely overlaps with the +// given range ("end" is assumed to exclusive). +func (m *FileMetadata) ContainedWithinSpan(cmp Compare, start, end []byte) bool { + lowerCmp, upperCmp := cmp(m.Smallest.UserKey, start), cmp(m.Largest.UserKey, end) + return lowerCmp >= 0 && (upperCmp < 0 || (upperCmp == 0 && m.Largest.IsExclusiveSentinel())) +} + // ContainsKeyType returns whether or not the file contains keys of the provided // type. func (m *FileMetadata) ContainsKeyType(kt KeyType) bool { @@ -466,6 +711,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string { if !verbose { return b.String() } + fmt.Fprintf(&b, " seqnums:[%d-%d]", m.SmallestSeqNum, m.LargestSeqNum) if m.HasPointKeys { fmt.Fprintf(&b, " points:[%s-%s]", m.SmallestPointKey.Pretty(format), m.LargestPointKey.Pretty(format)) @@ -479,9 +725,9 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string { // ParseFileMetadataDebug parses a FileMetadata from its DebugString // representation. -func ParseFileMetadataDebug(s string) (m FileMetadata, err error) { +func ParseFileMetadataDebug(s string) (*FileMetadata, error) { // Split lines of the form: - // 000000:[a#0,SET-z#0,SET] points:[...] ranges:[...] + // 000000:[a#0,SET-z#0,SET] seqnums:[5-5] points:[...] ranges:[...] fields := strings.FieldsFunc(s, func(c rune) bool { switch c { case ':', '[', '-', ']': @@ -491,10 +737,24 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) { } }) if len(fields)%3 != 0 { - return m, errors.Newf("malformed input: %s", s) + return nil, errors.Newf("malformed input: %s", s) } + m := &FileMetadata{} for len(fields) > 0 { prefix := fields[0] + if prefix == "seqnums" { + smallestSeqNum, err := strconv.ParseUint(fields[1], 10, 64) + if err != nil { + return m, errors.Newf("malformed input: %s: %s", s, err) + } + largestSeqNum, err := strconv.ParseUint(fields[2], 10, 64) + if err != nil { + return m, errors.Newf("malformed input: %s: %s", s, err) + } + m.SmallestSeqNum, m.LargestSeqNum = smallestSeqNum, largestSeqNum + fields = fields[3:] + continue + } smallest := base.ParsePrettyInternalKey(fields[1]) largest := base.ParsePrettyInternalKey(fields[2]) switch prefix { @@ -522,7 +782,8 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) { m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest m.HasPointKeys = true } - return + m.InitPhysicalBacking() + return m, nil } // Validate validates the metadata for consistency with itself, returning an @@ -584,6 +845,11 @@ func (m *FileMetadata) Validate(cmp Compare, formatKey base.FormatKey) error { } } + // Ensure that FileMetadata.Init was called. + if m.FileBacking == nil { + return base.CorruptionErrorf("file metadata FileBacking not set") + } + return nil } @@ -756,6 +1022,9 @@ func NewVersion( } else { v.Levels[l].tree.cmp = btreeCmpSmallestKey(cmp) } + for _, f := range files[l] { + v.Levels[l].totalSize += f.Size + } } if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil { panic(err) @@ -788,7 +1057,7 @@ func NewVersion( // key in a higher level table that has both the same user key and a higher // sequence number. type Version struct { - refs int32 + refs atomic.Int32 // The level 0 sstables are organized in a series of sublevels. Similar to // the seqnum invariant in normal levels, there is no internal key in a @@ -805,7 +1074,7 @@ type Version struct { // usage. // // L0Sublevels.Levels contains L0 files ordered by sublevels. All the files - // in Files[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to + // in Levels[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to // a reference to that slice, as that slice is necessary for iterator // creation and needs to outlast L0Sublevels. L0Sublevels *L0Sublevels @@ -820,7 +1089,7 @@ type Version struct { // The callback to invoke when the last reference to a version is // removed. Will be called with list.mu held. - Deleted func(obsolete []*FileMetadata) + Deleted func(obsolete []*FileBacking) // Stats holds aggregated stats about the version maintained from // version to version. @@ -849,15 +1118,21 @@ func (v *Version) DebugString(format base.FormatKey) string { return v.string(format, true) } +func describeSublevels(format base.FormatKey, verbose bool, sublevels []LevelSlice) string { + var buf bytes.Buffer + for sublevel := len(sublevels) - 1; sublevel >= 0; sublevel-- { + fmt.Fprintf(&buf, "0.%d:\n", sublevel) + sublevels[sublevel].Each(func(f *FileMetadata) { + fmt.Fprintf(&buf, " %s\n", f.DebugString(format, verbose)) + }) + } + return buf.String() +} + func (v *Version) string(format base.FormatKey, verbose bool) string { var buf bytes.Buffer if len(v.L0SublevelFiles) > 0 { - for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- { - fmt.Fprintf(&buf, "0.%d:\n", sublevel) - v.L0SublevelFiles[sublevel].Each(func(f *FileMetadata) { - fmt.Fprintf(&buf, " %s\n", f.DebugString(format, verbose)) - }) - } + fmt.Fprintf(&buf, "%s", describeSublevels(format, verbose, v.L0SublevelFiles)) } for level := 1; level < NumLevels; level++ { if v.Levels[level].Empty() { @@ -898,7 +1173,7 @@ func ParseVersionDebug( m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest m.HasPointKeys = true } - files[level] = append(files[level], &m) + files[level] = append(files[level], m) } } // Reverse the order of L0 files. This ensures we construct the same @@ -912,12 +1187,12 @@ func ParseVersionDebug( // Refs returns the number of references to the version. func (v *Version) Refs() int32 { - return atomic.LoadInt32(&v.refs) + return v.refs.Load() } // Ref increments the version refcount. func (v *Version) Ref() { - atomic.AddInt32(&v.refs, 1) + v.refs.Add(1) } // Unref decrements the version refcount. If the last reference to the version @@ -925,12 +1200,11 @@ func (v *Version) Ref() { // Deleted callback is invoked. Requires that the VersionList mutex is NOT // locked. func (v *Version) Unref() { - if atomic.AddInt32(&v.refs, -1) == 0 { - obsolete := v.unrefFiles() + if v.refs.Add(-1) == 0 { l := v.list l.mu.Lock() l.Remove(v) - v.Deleted(obsolete) + v.Deleted(v.unrefFiles()) l.mu.Unlock() } } @@ -940,14 +1214,14 @@ func (v *Version) Unref() { // the Deleted callback is invoked. Requires that the VersionList mutex is // already locked. func (v *Version) UnrefLocked() { - if atomic.AddInt32(&v.refs, -1) == 0 { + if v.refs.Add(-1) == 0 { v.list.Remove(v) v.Deleted(v.unrefFiles()) } } -func (v *Version) unrefFiles() []*FileMetadata { - var obsolete []*FileMetadata +func (v *Version) unrefFiles() []*FileBacking { + var obsolete []*FileBacking for _, lm := range v.Levels { obsolete = append(obsolete, lm.release()...) } @@ -1079,16 +1353,20 @@ func (v *Version) Overlaps( // CheckOrdering checks that the files are consistent with respect to // increasing file numbers (for level 0 files) and increasing and non- // overlapping internal key ranges (for level non-0 files). -func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error { +func (v *Version) CheckOrdering( + cmp Compare, format base.FormatKey, order OrderingInvariants, +) error { for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- { sublevelIter := v.L0SublevelFiles[sublevel].Iter() - if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter); err != nil { + // Sublevels have NEVER allowed split user keys, so we can pass + // ProhibitSplitUserKeys. + if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter, ProhibitSplitUserKeys); err != nil { return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format)) } } for level, lm := range v.Levels { - if err := CheckOrdering(cmp, format, Level(level), lm.Iter()); err != nil { + if err := CheckOrdering(cmp, format, Level(level), lm.Iter(), order); err != nil { return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format)) } } @@ -1157,10 +1435,34 @@ func (l *VersionList) Remove(v *Version) { v.list = nil // avoid memory leaks } +// OrderingInvariants dictates the file ordering invariants active. +type OrderingInvariants int8 + +const ( + // ProhibitSplitUserKeys indicates that adjacent files within a level cannot + // contain the same user key. + ProhibitSplitUserKeys OrderingInvariants = iota + // AllowSplitUserKeys indicates that adjacent files within a level may + // contain the same user key. This is only allowed by historical format + // major versions. + // + // TODO(jackson): Remove. + AllowSplitUserKeys +) + // CheckOrdering checks that the files are consistent with respect to // seqnums (for level 0 files -- see detailed comment below) and increasing and non- // overlapping internal key ranges (for non-level 0 files). -func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelIterator) error { +// +// The ordering field may be passed AllowSplitUserKeys to allow adjacent files that are both +// inclusive of the same user key. Pebble no longer creates version edits +// installing such files, and Pebble databases with sufficiently high format +// major version should no longer have any such files within their LSM. +// TODO(jackson): Remove AllowSplitUserKeys when we remove support for the +// earlier format major versions. +func CheckOrdering( + cmp Compare, format base.FormatKey, level Level, files LevelIterator, ordering OrderingInvariants, +) error { // The invariants to check for L0 sublevels are the same as the ones to // check for all other levels. However, if L0 is not organized into // sublevels, or if all L0 files are being passed in, we do the legacy L0 @@ -1238,11 +1540,29 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelI prev.Smallest.Pretty(format), prev.Largest.Pretty(format), f.Smallest.Pretty(format), f.Largest.Pretty(format)) } - if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 { - return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]", - errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum), - prev.Smallest.Pretty(format), prev.Largest.Pretty(format), - f.Smallest.Pretty(format), f.Largest.Pretty(format)) + + // What's considered "overlapping" is dependent on the format + // major version. If ordering=ProhibitSplitUserKeys, then both + // files cannot contain keys with the same user keys. If the + // bounds have the same user key, the previous file's boundary + // must have a Trailer indicating that it's exclusive. + switch ordering { + case AllowSplitUserKeys: + if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 { + return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]", + errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum), + prev.Smallest.Pretty(format), prev.Largest.Pretty(format), + f.Smallest.Pretty(format), f.Largest.Pretty(format)) + } + case ProhibitSplitUserKeys: + if v := cmp(prev.Largest.UserKey, f.Smallest.UserKey); v > 0 || (v == 0 && !prev.Largest.IsExclusiveSentinel()) { + return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]", + errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum), + prev.Smallest.Pretty(format), prev.Largest.Pretty(format), + f.Smallest.Pretty(format), f.Largest.Pretty(format)) + } + default: + panic("unreachable") } } } diff --git a/vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go b/vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go index ed591f8..d7853c2 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go +++ b/vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go @@ -11,11 +11,11 @@ import ( "fmt" "io" "sort" - "sync/atomic" "time" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" ) // TODO(peter): describe the MANIFEST file format, independently of the C++ @@ -51,7 +51,9 @@ const ( tagMaxColumnFamily = 203 // Pebble tags. - tagNewFile5 = 104 // Range keys. + tagNewFile5 = 104 // Range keys. + tagCreatedBackingTable = 105 + tagRemovedBackingTable = 106 // The custom tags sub-format used by tagNewFile4 and above. customTagTerminate = 1 @@ -59,6 +61,7 @@ const ( customTagCreationTime = 6 customTagPathID = 65 customTagNonSafeIgnoreMask = 1 << 6 + customTagVirtual = 66 ) // DeletedFileEntry holds the state for a file deletion from a level. The file @@ -73,6 +76,9 @@ type DeletedFileEntry struct { type NewFileEntry struct { Level int Meta *FileMetadata + // BackingFileNum is only set during manifest replay, and only for virtual + // sstables. + BackingFileNum base.DiskFileNum } // VersionEdit holds the state for an edit to a Version along with other @@ -110,9 +116,42 @@ type VersionEdit struct { // found that there was no overlapping file at the higher level). DeletedFiles map[DeletedFileEntry]*FileMetadata NewFiles []NewFileEntry + // CreatedBackingTables can be used to preserve the FileBacking associated + // with a physical sstable. This is useful when virtual sstables in the + // latest version are reconstructed during manifest replay, and we also need + // to reconstruct the FileBacking which is required by these virtual + // sstables. + // + // INVARIANT: The FileBacking associated with a physical sstable must only + // be added as a backing file in the same version edit where the physical + // sstable is first virtualized. This means that the physical sstable must + // be present in DeletedFiles and that there must be at least one virtual + // sstable with the same FileBacking as the physical sstable in NewFiles. A + // file must be present in CreatedBackingTables in exactly one version edit. + // The physical sstable associated with the FileBacking must also not be + // present in NewFiles. + CreatedBackingTables []*FileBacking + // RemovedBackingTables is used to remove the FileBacking associated with a + // virtual sstable. Note that a backing sstable can be removed as soon as + // there are no virtual sstables in the latest version which are using the + // backing sstable, but the backing sstable doesn't necessarily have to be + // removed atomically with the version edit which removes the last virtual + // sstable associated with the backing sstable. The removal can happen in a + // future version edit. + // + // INVARIANT: A file must only be added to RemovedBackingTables if it was + // added to CreateBackingTables in a prior version edit. The same version + // edit also cannot have the same file present in both CreateBackingTables + // and RemovedBackingTables. A file must be present in RemovedBackingTables + // in exactly one version edit. + RemovedBackingTables []base.DiskFileNum } // Decode decodes an edit from the specified reader. +// +// Note that the Decode step will not set the FileBacking for virtual sstables +// and the responsibility is left to the caller. However, the Decode step will +// populate the NewFileEntry.BackingFileNum in VersionEdit.NewFiles. func (v *VersionEdit) Decode(r io.Reader) error { br, ok := r.(byteReader) if !ok { @@ -165,6 +204,28 @@ func (v *VersionEdit) Decode(r io.Reader) error { } // NB: RocksDB does not use compaction pointers anymore. + case tagRemovedBackingTable: + n, err := d.readUvarint() + if err != nil { + return err + } + v.RemovedBackingTables = append( + v.RemovedBackingTables, base.FileNum(n).DiskFileNum(), + ) + case tagCreatedBackingTable: + dfn, err := d.readUvarint() + if err != nil { + return err + } + size, err := d.readUvarint() + if err != nil { + return err + } + fileBacking := &FileBacking{ + DiskFileNum: base.FileNum(dfn).DiskFileNum(), + Size: size, + } + v.CreatedBackingTables = append(v.CreatedBackingTables, fileBacking) case tagDeletedFile: level, err := d.readLevel() if err != nil { @@ -270,6 +331,10 @@ func (v *VersionEdit) Decode(r io.Reader) error { } var markedForCompaction bool var creationTime uint64 + virtualState := struct { + virtual bool + backingFileNum uint64 + }{} if tag == tagNewFile4 || tag == tagNewFile5 { for { customTag, err := d.readUvarint() @@ -278,7 +343,16 @@ func (v *VersionEdit) Decode(r io.Reader) error { } if customTag == customTagTerminate { break + } else if customTag == customTagVirtual { + virtualState.virtual = true + n, err := d.readUvarint() + if err != nil { + return err + } + virtualState.backingFileNum = n + continue } + field, err := d.readBytes() if err != nil { return err @@ -314,6 +388,7 @@ func (v *VersionEdit) Decode(r io.Reader) error { SmallestSeqNum: smallestSeqNum, LargestSeqNum: largestSeqNum, MarkedForCompaction: markedForCompaction, + Virtual: virtualState.virtual, } if tag != tagNewFile5 { // no range keys present m.SmallestPointKey = base.DecodeInternalKey(smallestPointKey) @@ -345,10 +420,18 @@ func (v *VersionEdit) Decode(r io.Reader) error { } } m.boundsSet = true - v.NewFiles = append(v.NewFiles, NewFileEntry{ + if !virtualState.virtual { + m.InitPhysicalBacking() + } + + nfe := NewFileEntry{ Level: level, Meta: m, - }) + } + if virtualState.virtual { + nfe.BackingFileNum = base.FileNum(virtualState.backingFileNum).DiskFileNum() + } + v.NewFiles = append(v.NewFiles, nfe) case tagPrevLogNumber: n, err := d.readUvarint() @@ -367,8 +450,7 @@ func (v *VersionEdit) Decode(r io.Reader) error { return nil } -// String implements fmt.Stringer for a VersionEdit. -func (v *VersionEdit) String() string { +func (v *VersionEdit) string(verbose bool, fmtKey base.FormatKey) string { var buf bytes.Buffer if v.ComparerName != "" { fmt.Fprintf(&buf, " comparer: %s", v.ComparerName) @@ -399,7 +481,12 @@ func (v *VersionEdit) String() string { fmt.Fprintf(&buf, " deleted: L%d %s\n", df.Level, df.FileNum) } for _, nf := range v.NewFiles { - fmt.Fprintf(&buf, " added: L%d %s", nf.Level, nf.Meta.String()) + fmt.Fprintf(&buf, " added: L%d", nf.Level) + if verbose { + fmt.Fprintf(&buf, " %s", nf.Meta.DebugString(fmtKey, true /* verbose */)) + } else { + fmt.Fprintf(&buf, " %s", nf.Meta.String()) + } if nf.Meta.CreationTime != 0 { fmt.Fprintf(&buf, " (%s)", time.Unix(nf.Meta.CreationTime, 0).UTC().Format(time.RFC3339)) @@ -409,6 +496,16 @@ func (v *VersionEdit) String() string { return buf.String() } +// DebugString is a more verbose version of String(). Use this in tests. +func (v *VersionEdit) DebugString(fmtKey base.FormatKey) string { + return v.string(true /* verbose */, fmtKey) +} + +// String implements fmt.Stringer for a VersionEdit. +func (v *VersionEdit) String() string { + return v.string(false /* verbose */, base.DefaultFormatter) +} + // Encode encodes an edit to the specified writer. func (v *VersionEdit) Encode(w io.Writer) error { e := versionEditEncoder{new(bytes.Buffer)} @@ -429,6 +526,15 @@ func (v *VersionEdit) Encode(w io.Writer) error { e.writeUvarint(tagNextFileNumber) e.writeUvarint(uint64(v.NextFileNum)) } + for _, dfn := range v.RemovedBackingTables { + e.writeUvarint(tagRemovedBackingTable) + e.writeUvarint(uint64(dfn.FileNum())) + } + for _, fileBacking := range v.CreatedBackingTables { + e.writeUvarint(tagCreatedBackingTable) + e.writeUvarint(uint64(fileBacking.DiskFileNum.FileNum())) + e.writeUvarint(fileBacking.Size) + } // RocksDB requires LastSeqNum to be encoded for the first MANIFEST entry, // even though its value is zero. We detect this by encoding LastSeqNum when // ComparerName is set. @@ -442,7 +548,7 @@ func (v *VersionEdit) Encode(w io.Writer) error { e.writeUvarint(uint64(x.FileNum)) } for _, x := range v.NewFiles { - customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0 + customFields := x.Meta.MarkedForCompaction || x.Meta.CreationTime != 0 || x.Meta.Virtual var tag uint64 switch { case x.Meta.HasRangeKeys: @@ -495,6 +601,10 @@ func (v *VersionEdit) Encode(w io.Writer) error { e.writeUvarint(customTagNeedsCompaction) e.writeBytes([]byte{1}) } + if x.Meta.Virtual { + e.writeUvarint(customTagVirtual) + e.writeUvarint(uint64(x.Meta.FileBacking.DiskFileNum.FileNum())) + } e.writeUvarint(customTagTerminate) } } @@ -502,6 +612,7 @@ func (v *VersionEdit) Encode(w io.Writer) error { return err } +// versionEditDecoder should be used to decode version edits. type versionEditDecoder struct { byteReader } @@ -581,10 +692,30 @@ func (e versionEditEncoder) writeUvarint(u uint64) { // BulkVersionEdit summarizes the files added and deleted from a set of version // edits. +// +// INVARIANTS: +// No file can be added to a level more than once. This is true globally, and +// also true for all of the calls to Accumulate for a single bulk version edit. +// +// No file can be removed from a level more than once. This is true globally, +// and also true for all of the calls to Accumulate for a single bulk version +// edit. +// +// A file must not be added and removed from a given level in the same version +// edit. +// +// A file that is being removed from a level must have been added to that level +// before (in a prior version edit). Note that a given file can be deleted from +// a level and added to another level in a single version edit type BulkVersionEdit struct { - Added [NumLevels][]*FileMetadata + Added [NumLevels]map[base.FileNum]*FileMetadata Deleted [NumLevels]map[base.FileNum]*FileMetadata + // AddedFileBacking is a map to support lookup so that we can populate the + // FileBacking of virtual sstables during manifest replay. + AddedFileBacking map[base.DiskFileNum]*FileBacking + RemovedFileBacking []base.DiskFileNum + // AddedByFileNum maps file number to file metadata for all added files // from accumulated version edits. AddedByFileNum is only populated if set // to non-nil by a caller. It must be set to non-nil when replaying @@ -603,6 +734,19 @@ type BulkVersionEdit struct { // Accumulate adds the file addition and deletions in the specified version // edit to the bulk edit's internal state. +// +// INVARIANTS: +// If a file is added to a given level in a call to Accumulate and then removed +// from that level in a subsequent call, the file will not be present in the +// resulting BulkVersionEdit.Deleted for that level. +// +// After accumulation of version edits, the bulk version edit may have +// information about a file which has been deleted from a level, but it may +// not have information about the same file added to the same level. The add +// could've occurred as part of a previous bulk version edit. In this case, +// the deleted file must be present in BulkVersionEdit.Deleted, at the end +// of the accumulation, because we need to decrease the refcount of the +// deleted file in Apply. func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error { for df, m := range ve.DeletedFiles { dmap := b.Deleted[df.Level] @@ -624,7 +768,28 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error { if m.MarkedForCompaction { b.MarkedForCompactionCountDiff-- } - dmap[df.FileNum] = m + if _, ok := b.Added[df.Level][df.FileNum]; !ok { + dmap[df.FileNum] = m + } else { + // Present in b.Added for the same level. + delete(b.Added[df.Level], df.FileNum) + } + } + + // Generate state for Added backing files. Note that these must be generated + // before we loop through the NewFiles, because we need to populate the + // FileBackings which might be used by the NewFiles loop. + if b.AddedFileBacking == nil { + b.AddedFileBacking = make(map[base.DiskFileNum]*FileBacking) + } + for _, fb := range ve.CreatedBackingTables { + if _, ok := b.AddedFileBacking[fb.DiskFileNum]; ok { + // There is already a FileBacking associated with fb.DiskFileNum. + // This should never happen. There must always be only one FileBacking + // associated with a backing sstable. + panic(fmt.Sprintf("pebble: duplicate file backing %s", fb.DiskFileNum.String())) + } + b.AddedFileBacking[fb.DiskFileNum] = fb } for _, nf := range ve.NewFiles { @@ -635,7 +800,21 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error { return base.CorruptionErrorf("pebble: file deleted L%d.%s before it was inserted", nf.Level, nf.Meta.FileNum) } } - b.Added[nf.Level] = append(b.Added[nf.Level], nf.Meta) + if nf.Meta.Virtual && nf.Meta.FileBacking == nil { + // FileBacking for a virtual sstable must only be nil if we're performing + // manifest replay. + nf.Meta.FileBacking = b.AddedFileBacking[nf.BackingFileNum] + if nf.Meta.FileBacking == nil { + return errors.Errorf("FileBacking for virtual sstable must not be nil") + } + } else if nf.Meta.FileBacking == nil { + return errors.Errorf("Added file L%d.%s's has no FileBacking", nf.Level, nf.Meta.FileNum) + } + + if b.Added[nf.Level] == nil { + b.Added[nf.Level] = make(map[base.FileNum]*FileMetadata) + } + b.Added[nf.Level][nf.Meta.FileNum] = nf.Meta if b.AddedByFileNum != nil { b.AddedByFileNum[nf.Meta.FileNum] = nf.Meta } @@ -643,36 +822,101 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error { b.MarkedForCompactionCountDiff++ } } + + // Since a file can be removed from backing files in exactly one version + // edit it is safe to just append without any de-duplication. + b.RemovedFileBacking = append(b.RemovedFileBacking, ve.RemovedBackingTables...) + return nil } +// AccumulateIncompleteAndApplySingleVE should be called if a single version edit +// is to be applied to the provided curr Version and if the caller needs to +// update the versionSet.zombieTables map. This function exists separately from +// BulkVersionEdit.Apply because it is easier to reason about properties +// regarding BulkVersionedit.Accumulate/Apply and zombie table generation, if we +// know that exactly one version edit is being accumulated. +// +// Note that the version edit passed into this function may be incomplete +// because compactions don't have the ref counting information necessary to +// populate VersionEdit.RemovedBackingTables. This function will complete such a +// version edit by populating RemovedBackingTables. +// +// Invariant: Any file being deleted through ve must belong to the curr Version. +// We can't have a delete for some arbitrary file which does not exist in curr. +func AccumulateIncompleteAndApplySingleVE( + ve *VersionEdit, + curr *Version, + cmp Compare, + formatKey base.FormatKey, + flushSplitBytes int64, + readCompactionRate int64, + backingStateMap map[base.DiskFileNum]*FileBacking, + addBackingFunc func(*FileBacking), + removeBackingFunc func(base.DiskFileNum), + orderingInvariants OrderingInvariants, +) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) { + if len(ve.RemovedBackingTables) != 0 { + panic("pebble: invalid incomplete version edit") + } + var b BulkVersionEdit + err := b.Accumulate(ve) + if err != nil { + return nil, nil, err + } + zombies = make(map[base.DiskFileNum]uint64) + v, err := b.Apply( + curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies, orderingInvariants, + ) + if err != nil { + return nil, nil, err + } + + for _, s := range b.AddedFileBacking { + addBackingFunc(s) + } + + for fileNum := range zombies { + if _, ok := backingStateMap[fileNum]; ok { + // This table was backing some virtual sstable in the latest version, + // but is now a zombie. We add RemovedBackingTables entries for + // these, before the version edit is written to disk. + ve.RemovedBackingTables = append( + ve.RemovedBackingTables, fileNum, + ) + removeBackingFunc(fileNum) + } + } + return v, zombies, nil +} + // Apply applies the delta b to the current version to produce a new // version. The new version is consistent with respect to the comparer cmp. // // curr may be nil, which is equivalent to a pointer to a zero version. // -// On success, a map of zombie files containing the file numbers and sizes of -// deleted files is returned. These files are considered zombies because they -// are no longer referenced by the returned Version, but cannot be deleted from -// disk as they are still in use by the incoming Version. +// On success, if a non-nil zombies map is provided to Apply, the map is updated +// with file numbers and files sizes of deleted files. These files are +// considered zombies because they are no longer referenced by the returned +// Version, but cannot be deleted from disk as they are still in use by the +// incoming Version. func (b *BulkVersionEdit) Apply( curr *Version, cmp Compare, formatKey base.FormatKey, flushSplitBytes int64, readCompactionRate int64, -) (_ *Version, zombies map[base.FileNum]uint64, _ error) { - addZombie := func(fileNum base.FileNum, size uint64) { - if zombies == nil { - zombies = make(map[base.FileNum]uint64) - } - zombies[fileNum] = size - } - // The remove zombie function is used to handle tables that are moved from - // one level to another during a version edit (i.e. a "move" compaction). - removeZombie := func(fileNum base.FileNum) { + zombies map[base.DiskFileNum]uint64, + orderingInvariants OrderingInvariants, +) (*Version, error) { + addZombie := func(state *FileBacking) { if zombies != nil { - delete(zombies, fileNum) + zombies[state.DiskFileNum] = state.Size + } + } + removeZombie := func(state *FileBacking) { + if zombies != nil { + delete(zombies, state.DiskFileNum) } } @@ -684,7 +928,7 @@ func (b *BulkVersionEdit) Apply( } v.Stats.MarkedForCompaction += b.MarkedForCompactionCountDiff if v.Stats.MarkedForCompaction < 0 { - return nil, nil, base.CorruptionErrorf("pebble: version marked for compaction count negative") + return nil, base.CorruptionErrorf("pebble: version marked for compaction count negative") } for level := range v.Levels { @@ -705,7 +949,7 @@ func (b *BulkVersionEdit) Apply( // Initialize L0Sublevels. if curr == nil || curr.L0Sublevels == nil { if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil { - return nil, nil, errors.Wrap(err, "pebble: internal error") + return nil, errors.Wrap(err, "pebble: internal error") } } else { v.L0Sublevels = curr.L0Sublevels @@ -718,47 +962,70 @@ func (b *BulkVersionEdit) Apply( // Some edits on this level. lm := &v.Levels[level] lmRange := &v.RangeKeyLevels[level] - addedFiles := b.Added[level] - deletedMap := b.Deleted[level] - if n := v.Levels[level].Len() + len(addedFiles); n == 0 { - return nil, nil, base.CorruptionErrorf( + + addedFilesMap := b.Added[level] + deletedFilesMap := b.Deleted[level] + if n := v.Levels[level].Len() + len(addedFilesMap); n == 0 { + return nil, base.CorruptionErrorf( "pebble: internal error: No current or added files but have deleted files: %d", - errors.Safe(len(deletedMap))) + errors.Safe(len(deletedFilesMap))) } - // NB: addedFiles may be empty and it also is not necessarily - // internally consistent: it does not reflect deletions in deletedMap. + // NB: addedFilesMap may be empty. If a file is present in addedFilesMap + // for a level, it won't be present in deletedFilesMap for the same + // level. - for _, f := range deletedMap { - addZombie(f.FileNum, f.Size) - if obsolete := v.Levels[level].tree.Delete(f); obsolete { + for _, f := range deletedFilesMap { + if obsolete := v.Levels[level].remove(f); obsolete { // Deleting a file from the B-Tree may decrement its // reference count. However, because we cloned the // previous level's B-Tree, this should never result in a // file's reference count dropping to zero. err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during B-Tree removal", level, f.FileNum) - return nil, nil, err + return nil, err } if f.HasRangeKeys { - if obsolete := v.RangeKeyLevels[level].tree.Delete(f); obsolete { + if obsolete := v.RangeKeyLevels[level].remove(f); obsolete { // Deleting a file from the B-Tree may decrement its // reference count. However, because we cloned the // previous level's B-Tree, this should never result in a // file's reference count dropping to zero. err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during range-key B-Tree removal", level, f.FileNum) - return nil, nil, err + return nil, err } } + + // Note that a backing sst will only become a zombie if the + // references to it in the latest version is 0. We will remove the + // backing sst from the zombie list in the next loop if one of the + // addedFiles in any of the levels is referencing the backing sst. + // This is possible if a physical sstable is virtualized, or if it + // is moved. + latestRefCount := f.LatestRefs() + if latestRefCount <= 0 { + // If a file is present in deletedFilesMap for a level, then it + // must have already been added to the level previously, which + // means that its latest ref count cannot be 0. + err := errors.Errorf("pebble: internal error: incorrect latestRefs reference counting for file", f.FileNum) + return nil, err + } else if f.LatestUnref() == 0 { + addZombie(f.FileBacking) + } } + addedFiles := make([]*FileMetadata, 0, len(addedFilesMap)) + for _, f := range addedFilesMap { + addedFiles = append(addedFiles, f) + } + // Sort addedFiles by file number. This isn't necessary, but tests which + // replay invalid manifests check the error output, and the error output + // depends on the order in which files are added to the btree. + sort.Slice(addedFiles, func(i, j int) bool { + return addedFiles[i].FileNum < addedFiles[j].FileNum + }) + var sm, la *FileMetadata for _, f := range addedFiles { - if _, ok := deletedMap[f.FileNum]; ok { - // Already called addZombie on this file in the preceding - // loop, so we don't need to do it here. - continue - } - // NB: allowedSeeks is used for read triggered compactions. It is set using // Options.Experimental.ReadCompactionRate which defaults to 32KB. var allowedSeeks int64 @@ -768,20 +1035,23 @@ func (b *BulkVersionEdit) Apply( if allowedSeeks < 100 { allowedSeeks = 100 } - atomic.StoreInt64(&f.Atomic.AllowedSeeks, allowedSeeks) + f.AllowedSeeks.Store(allowedSeeks) f.InitAllowedSeeks = allowedSeeks - err := lm.tree.Insert(f) + err := lm.insert(f) + // We're adding this file to the new version, so increment the + // latest refs count. + f.LatestRef() if err != nil { - return nil, nil, errors.Wrap(err, "pebble") + return nil, errors.Wrap(err, "pebble") } if f.HasRangeKeys { - err = lmRange.tree.Insert(f) + err = lmRange.insert(f) if err != nil { - return nil, nil, errors.Wrap(err, "pebble") + return nil, errors.Wrap(err, "pebble") } } - removeZombie(f.FileNum) + removeZombie(f.FileBacking) // Track the keys with the smallest and largest keys, so that we can // check consistency of the modified span. if sm == nil || base.InternalCompare(cmp, sm.Smallest, f.Smallest) > 0 { @@ -793,27 +1063,36 @@ func (b *BulkVersionEdit) Apply( } if level == 0 { - if curr != nil && curr.L0Sublevels != nil && len(deletedMap) == 0 { + if curr != nil && curr.L0Sublevels != nil && len(deletedFilesMap) == 0 { // Flushes and ingestions that do not delete any L0 files do not require // a regeneration of L0Sublevels from scratch. We can instead generate // it incrementally. var err error // AddL0Files requires addedFiles to be sorted in seqnum order. - addedFiles = append([]*FileMetadata(nil), addedFiles...) SortBySeqNum(addedFiles) v.L0Sublevels, err = curr.L0Sublevels.AddL0Files(addedFiles, flushSplitBytes, &v.Levels[0]) if errors.Is(err, errInvalidL0SublevelsOpt) { err = v.InitL0Sublevels(cmp, formatKey, flushSplitBytes) + } else if invariants.Enabled && err == nil { + copyOfSublevels, err := NewL0Sublevels(&v.Levels[0], cmp, formatKey, flushSplitBytes) + if err != nil { + panic(fmt.Sprintf("error when regenerating sublevels: %s", err)) + } + s1 := describeSublevels(base.DefaultFormatter, false /* verbose */, copyOfSublevels.Levels) + s2 := describeSublevels(base.DefaultFormatter, false /* verbose */, v.L0Sublevels.Levels) + if s1 != s2 { + panic(fmt.Sprintf("incremental L0 sublevel generation produced different output than regeneration: %s != %s", s1, s2)) + } } if err != nil { - return nil, nil, errors.Wrap(err, "pebble: internal error") + return nil, errors.Wrap(err, "pebble: internal error") } v.L0SublevelFiles = v.L0Sublevels.Levels } else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil { - return nil, nil, errors.Wrap(err, "pebble: internal error") + return nil, errors.Wrap(err, "pebble: internal error") } - if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil { - return nil, nil, errors.Wrap(err, "pebble: internal error") + if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter(), orderingInvariants); err != nil { + return nil, errors.Wrap(err, "pebble: internal error") } continue } @@ -833,10 +1112,10 @@ func (b *BulkVersionEdit) Apply( end.Prev() } }) - if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil { - return nil, nil, errors.Wrap(err, "pebble: internal error") + if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter(), orderingInvariants); err != nil { + return nil, errors.Wrap(err, "pebble: internal error") } } } - return v, zombies, nil + return v, nil } diff --git a/vendor/github.com/cockroachdb/pebble/internal/private/sstable.go b/vendor/github.com/cockroachdb/pebble/internal/private/sstable.go index fb38139..780dd56 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/private/sstable.go +++ b/vendor/github.com/cockroachdb/pebble/internal/private/sstable.go @@ -8,7 +8,7 @@ import "github.com/cockroachdb/pebble/internal/base" // SSTableCacheOpts is a hook for specifying cache options to // sstable.NewReader. -var SSTableCacheOpts func(cacheID uint64, fileNum base.FileNum) interface{} +var SSTableCacheOpts func(cacheID uint64, fileNum base.DiskFileNum) interface{} // SSTableRawTombstonesOpt is a sstable.Reader option for disabling // fragmentation of the range tombstones returned by diff --git a/vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go b/vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go index a25b64d..c0456bb 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go +++ b/vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go @@ -52,15 +52,15 @@ import ( // │ // ╰── .Next type UserIteratorConfig struct { - snapshot uint64 - comparer *base.Comparer - miter keyspan.MergingIter - biter keyspan.BoundedIter - diter keyspan.DefragmentingIter - liters [manifest.NumLevels]keyspan.LevelIter - litersUsed int - onlySets bool - bufs *Buffers + snapshot uint64 + comparer *base.Comparer + miter keyspan.MergingIter + biter keyspan.BoundedIter + diter keyspan.DefragmentingIter + liters [manifest.NumLevels]keyspan.LevelIter + litersUsed int + internalKeys bool + bufs *Buffers } // Buffers holds various buffers used for range key iteration. They're exposed @@ -79,9 +79,10 @@ func (bufs *Buffers) PrepareForReuse() { // Init initializes the range key iterator stack for user iteration. The // resulting fragment iterator applies range key semantics, defragments spans -// according to their user-observable state and, if onlySets = true, removes all +// according to their user-observable state and, if !internalKeys, removes all // Keys other than RangeKeySets describing the current state of range keys. The -// resulting spans contain Keys sorted by Suffix. +// resulting spans contain Keys sorted by suffix (unless internalKeys is true, +// in which case they remain sorted by trailer descending). // // The snapshot sequence number parameter determines which keys are visible. Any // keys not visible at the provided snapshot are ignored. @@ -91,16 +92,20 @@ func (ui *UserIteratorConfig) Init( lower, upper []byte, hasPrefix *bool, prefix *[]byte, - onlySets bool, + internalKeys bool, bufs *Buffers, iters ...keyspan.FragmentIterator, ) keyspan.FragmentIterator { ui.snapshot = snapshot ui.comparer = comparer - ui.onlySets = onlySets + ui.internalKeys = internalKeys ui.miter.Init(comparer.Compare, ui, &bufs.merging, iters...) ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix) - ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting) + if internalKeys { + ui.diter.Init(comparer, &ui.biter, keyspan.DefragmentInternal, keyspan.StaticDefragmentReducer, &bufs.defragmenting) + } else { + ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting) + } ui.litersUsed = 0 ui.bufs = bufs return &ui.diter @@ -136,7 +141,8 @@ func (ui *UserIteratorConfig) SetBounds(lower, upper []byte) { // of unset keys, removal of keys overwritten by a set at the same suffix, etc) // and then non-RangeKeySet keys are removed. The resulting transformed spans // only contain RangeKeySets describing the state visible at the provided -// sequence number, and hold their Keys sorted by Suffix. +// sequence number, and hold their Keys sorted by Suffix (except if internalKeys +// is true, then keys remain sorted by trailer. func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error { // Apply shadowing of keys. dst.Start = s.Start @@ -148,9 +154,16 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k if err := coalesce(ui.comparer.Equal, &ui.bufs.sortBuf, ui.snapshot, s.Keys); err != nil { return err } - // During user iteration over range keys, unsets and deletes don't matter. - // Remove them if onlySets = true. This step helps logical defragmentation - // during iteration. + if ui.internalKeys { + if s.KeysOrder != keyspan.ByTrailerDesc { + panic("unexpected key ordering in UserIteratorTransform with internalKeys = true") + } + dst.Keys = ui.bufs.sortBuf.Keys + keyspan.SortKeysByTrailer(&dst.Keys) + return nil + } + // During user iteration over range keys, unsets and deletes don't matter. This + // step helps logical defragmentation during iteration. keys := ui.bufs.sortBuf.Keys dst.Keys = dst.Keys[:0] for i := range keys { @@ -164,17 +177,11 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 { panic("pebble: keys unexpectedly not in ascending suffix order") } - if ui.onlySets { - // Skip. - continue - } - dst.Keys = append(dst.Keys, keys[i]) + // Skip. + continue case base.InternalKeyKindRangeKeyDelete: - if ui.onlySets { - // Skip. - continue - } - dst.Keys = append(dst.Keys, keys[i]) + // Skip. + continue default: return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind()) } @@ -193,6 +200,10 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k // sequence numbers). It's intended for use during user iteration, when the // wrapped keyspan iterator is merging spans across all levels of the LSM. func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.Span) bool { + // This method is not called with internalKeys = true. + if ui.internalKeys { + panic("unexpected call to ShouldDefragment with internalKeys = true") + } // This implementation must only be used on spans that have transformed by // ui.Transform. The transform applies shadowing, removes all keys besides // the resulting Sets and sorts the keys by suffix. Since shadowing has been @@ -208,8 +219,8 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S ret := true for i := range a.Keys { if invariants.Enabled { - if ui.onlySets && (a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet || - b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet) { + if a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet || + b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet { panic("pebble: unexpected non-RangeKeySet during defragmentation") } if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 || diff --git a/vendor/github.com/cockroachdb/pebble/internal/rate/rate.go b/vendor/github.com/cockroachdb/pebble/internal/rate/rate.go deleted file mode 100644 index 565626f..0000000 --- a/vendor/github.com/cockroachdb/pebble/internal/rate/rate.go +++ /dev/null @@ -1,387 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package rate provides a rate limiter. -package rate // import "github.com/cockroachdb/pebble/internal/rate" - -import ( - "context" - "math" - "sync" - "time" - - "github.com/cockroachdb/errors" -) - -// Limit defines the maximum frequency of some events. -// Limit is represented as number of events per second. -// A zero Limit allows no events. -type Limit float64 - -// Inf is the infinite rate limit; it allows all events (even if burst is zero). -const Inf = Limit(math.MaxFloat64) - -// Every converts a minimum time interval between events to a Limit. -func Every(interval time.Duration) Limit { - if interval <= 0 { - return Inf - } - return 1 / Limit(interval.Seconds()) -} - -// A Limiter controls how frequently events are allowed to happen. -// It implements a "token bucket" of size b, initially full and refilled -// at rate r tokens per second. -// Informally, in any large enough time interval, the Limiter limits the -// rate to r tokens per second, with a maximum burst size of b events. -// As a special case, if r == Inf (the infinite rate), b is ignored. -// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets. -// -// The zero value is a valid Limiter, but it will reject all events. -// Use NewLimiter to create non-zero Limiters. -// -// Limiter has three main methods, Allow, Reserve, and Wait. -// Most callers should use Wait. -// -// Each of the three methods consumes a single token. -// They differ in their behavior when no token is available. -// If no token is available, Allow returns false. -// If no token is available, Reserve returns a reservation for a future token -// and the amount of time the caller must wait before using it. -// If no token is available, Wait blocks until one can be obtained -// or its associated context.Context is canceled. -// -// The methods AllowN, ReserveN, and WaitN consume n tokens. -type Limiter struct { - limit Limit - burst int - - mu sync.Mutex - tokens float64 - // last is the last time the limiter's tokens field was updated - last time.Time - // lastEvent is the latest time of a rate-limited event (past or future) - lastEvent time.Time -} - -// Limit returns the maximum overall event rate. -func (lim *Limiter) Limit() Limit { - lim.mu.Lock() - defer lim.mu.Unlock() - return lim.limit -} - -// Burst returns the maximum burst size. Burst is the maximum number of tokens -// that can be consumed in a single call to Allow, Reserve, or Wait, so higher -// Burst values allow more events to happen at once. -// A zero Burst allows no events, unless limit == Inf. -func (lim *Limiter) Burst() int { - return lim.burst -} - -// NewLimiter returns a new Limiter that allows events up to rate r and permits -// bursts of at most b tokens. -func NewLimiter(r Limit, b int) *Limiter { - return &Limiter{ - limit: r, - burst: b, - } -} - -// Allow is shorthand for AllowN(time.Now(), 1). -func (lim *Limiter) Allow() bool { - return lim.AllowN(time.Now(), 1) -} - -// AllowN reports whether n events may happen at time now. -// Use this method if you intend to drop / skip events that exceed the rate limit. -// Otherwise use Reserve or Wait. -func (lim *Limiter) AllowN(now time.Time, n int) bool { - return lim.reserveN(now, n, 0).ok -} - -// A Reservation holds information about events that are permitted by a Limiter to happen after a delay. -// A Reservation may be canceled, which may enable the Limiter to permit additional events. -type Reservation struct { - ok bool - lim *Limiter - tokens int - timeToAct time.Time - // This is the Limit at reservation time, it can change later. - limit Limit -} - -// OK returns whether the limiter can provide the requested number of tokens -// within the maximum wait time. If OK is false, Delay returns InfDuration, and -// Cancel does nothing. -func (r *Reservation) OK() bool { - return r.ok -} - -// Delay is shorthand for DelayFrom(time.Now()). -func (r *Reservation) Delay() time.Duration { - return r.DelayFrom(time.Now()) -} - -// InfDuration is the duration returned by Delay when a Reservation is not OK. -const InfDuration = time.Duration(1<<63 - 1) - -// DelayFrom returns the duration for which the reservation holder must wait -// before taking the reserved action. Zero duration means act immediately. -// InfDuration means the limiter cannot grant the tokens requested in this -// Reservation within the maximum wait time. -func (r *Reservation) DelayFrom(now time.Time) time.Duration { - if !r.ok { - return InfDuration - } - delay := r.timeToAct.Sub(now) - if delay < 0 { - return 0 - } - return delay -} - -// Cancel is shorthand for CancelAt(time.Now()). -func (r *Reservation) Cancel() { - r.CancelAt(time.Now()) -} - -// CancelAt indicates that the reservation holder will not perform the reserved action -// and reverses the effects of this Reservation on the rate limit as much as possible, -// considering that other reservations may have already been made. -func (r *Reservation) CancelAt(now time.Time) { - if !r.ok { - return - } - - r.lim.mu.Lock() - defer r.lim.mu.Unlock() - - if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(now) { - return - } - - // calculate tokens to restore - // The duration between lim.lastEvent and r.timeToAct tells us how many tokens were reserved - // after r was obtained. These tokens should not be restored. - restoreTokens := float64(r.tokens) - r.limit.tokensFromDuration(r.lim.lastEvent.Sub(r.timeToAct)) - if restoreTokens <= 0 { - return - } - // advance time to now - now, _, tokens := r.lim.advance(now) - // calculate new number of tokens - tokens += restoreTokens - if burst := float64(r.lim.burst); tokens > burst { - tokens = burst - } - // update state - r.lim.last = now - r.lim.tokens = tokens - if r.timeToAct == r.lim.lastEvent { - prevEvent := r.timeToAct.Add(r.limit.durationFromTokens(float64(-r.tokens))) - if !prevEvent.Before(now) { - r.lim.lastEvent = prevEvent - } - } -} - -// Reserve is shorthand for ReserveN(time.Now(), 1). -func (lim *Limiter) Reserve() Reservation { - return lim.ReserveN(time.Now(), 1) -} - -// ReserveN returns a Reservation that indicates how long the caller must wait before n events happen. -// The Limiter takes this Reservation into account when allowing future events. -// ReserveN returns false if n exceeds the Limiter's burst size. -// Usage example: -// -// r := lim.ReserveN(time.Now(), 1) -// if !r.OK() { -// // Not allowed to act! Did you remember to set lim.burst to be > 0 ? -// return -// } -// time.Sleep(r.Delay()) -// Act() -// -// Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events. -// If you need to respect a deadline or cancel the delay, use Wait instead. -// To drop or skip events exceeding rate limit, use Allow instead. -func (lim *Limiter) ReserveN(now time.Time, n int) Reservation { - return lim.reserveN(now, n, InfDuration) -} - -// Wait is shorthand for WaitN(ctx, 1). -func (lim *Limiter) Wait(ctx context.Context) (err error) { - return lim.WaitN(ctx, 1) -} - -// WaitN blocks until lim permits n events to happen. -// It returns an error if n exceeds the Limiter's burst size, the Context is -// canceled, or the expected wait time exceeds the Context's Deadline. -func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) { - if n > lim.burst && lim.limit != Inf { - return errors.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", errors.Safe(n), errors.Safe(lim.burst)) - } - // Check if ctx is already cancelled - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - // Determine wait limit - now := time.Now() - waitLimit := InfDuration - if deadline, ok := ctx.Deadline(); ok { - waitLimit = deadline.Sub(now) - } - // Reserve - r := lim.reserveN(now, n, waitLimit) - if !r.ok { - return errors.Errorf("rate: Wait(n=%d) would exceed context deadline", errors.Safe(n)) - } - // Wait - d := r.DelayFrom(now) - if d <= 0 { - return nil - } - t := time.NewTimer(d) - defer t.Stop() - select { - case <-t.C: - // We can proceed. - return nil - case <-ctx.Done(): - // Context was canceled before we could proceed. Cancel the - // reservation, which may permit other events to proceed sooner. - r.Cancel() - return ctx.Err() - } -} - -// Delay is shorthand for DelayN(time.Now(), 1). -func (lim *Limiter) Delay() time.Duration { - return lim.DelayN(time.Now(), 1) -} - -// DelayN returns the delay to wait to permit n events to happen. Zero duration -// means act immediately. InfDuration means the limiter cannot grant the tokens -// requested within the maximum wait time. -func (lim *Limiter) DelayN(now time.Time, n int) time.Duration { - r := lim.reserveN(now, n, InfDuration) - return r.DelayFrom(now) -} - -// SetLimit is shorthand for SetLimitAt(time.Now(), newLimit). -func (lim *Limiter) SetLimit(newLimit Limit) { - lim.SetLimitAt(time.Now(), newLimit) -} - -// SetLimitAt sets a new Limit for the limiter. The new Limit, and Burst, may be violated -// or underutilized by those which reserved (using Reserve or Wait) but did not yet act -// before SetLimitAt was called. -func (lim *Limiter) SetLimitAt(now time.Time, newLimit Limit) { - lim.mu.Lock() - defer lim.mu.Unlock() - - now, _, tokens := lim.advance(now) - - lim.last = now - lim.tokens = tokens - lim.limit = newLimit -} - -// reserveN is a helper method for AllowN, ReserveN, and WaitN. -// maxFutureReserve specifies the maximum reservation wait duration allowed. -// reserveN returns Reservation, not *Reservation, to avoid allocation in AllowN and WaitN. -func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duration) Reservation { - lim.mu.Lock() - - if lim.limit == Inf { - lim.mu.Unlock() - return Reservation{ - ok: true, - lim: lim, - tokens: n, - timeToAct: now, - } - } - - now, last, tokens := lim.advance(now) - - // Calculate the remaining number of tokens resulting from the request. - tokens -= float64(n) - - // Calculate the wait duration - var waitDuration time.Duration - if tokens < 0 { - waitDuration = lim.limit.durationFromTokens(-tokens) - } - - // Decide result - ok := n <= lim.burst && waitDuration <= maxFutureReserve - - // Prepare reservation - r := Reservation{ - ok: ok, - lim: lim, - limit: lim.limit, - } - if ok { - r.tokens = n - r.timeToAct = now.Add(waitDuration) - } - - // Update state - if ok { - lim.last = now - lim.tokens = tokens - lim.lastEvent = r.timeToAct - } else { - lim.last = last - } - - lim.mu.Unlock() - return r -} - -// advance calculates and returns an updated state for lim resulting from the passage of time. -// lim is not changed. -func (lim *Limiter) advance( - now time.Time, -) (newNow time.Time, newLast time.Time, newTokens float64) { - last := lim.last - if now.Before(last) { - last = now - } - - // Avoid making delta overflow below when last is very old. - maxElapsed := lim.limit.durationFromTokens(float64(lim.burst) - lim.tokens) - elapsed := now.Sub(last) - if elapsed > maxElapsed { - elapsed = maxElapsed - } - - // Calculate the new number of tokens, due to time that passed. - delta := lim.limit.tokensFromDuration(elapsed) - tokens := lim.tokens + delta - if burst := float64(lim.burst); tokens > burst { - tokens = burst - } - - return now, last, tokens -} - -// durationFromTokens is a unit conversion function from the number of tokens to the duration -// of time it takes to accumulate them at a rate of limit tokens per second. -func (limit Limit) durationFromTokens(tokens float64) time.Duration { - seconds := tokens / float64(limit) - return time.Nanosecond * time.Duration(1e9*seconds) -} - -// tokensFromDuration is a unit conversion function from a time duration to the number of tokens -// which could be accumulated during that duration at a rate of limit tokens per second. -func (limit Limit) tokensFromDuration(d time.Duration) float64 { - return d.Seconds() * float64(limit) -} diff --git a/vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go b/vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go index 8ac89e2..fa7b10c 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go +++ b/vendor/github.com/cockroachdb/pebble/internal/testkeys/testkeys.go @@ -20,17 +20,27 @@ import ( "strings" "github.com/cockroachdb/pebble/internal/base" + "golang.org/x/exp/constraints" + "golang.org/x/exp/rand" ) const alpha = "abcdefghijklmnopqrstuvwxyz" const suffixDelim = '@' +var inverseAlphabet = make(map[byte]int64, len(alpha)) + +func init() { + for i := range alpha { + inverseAlphabet[alpha[i]] = int64(i) + } +} + // MaxSuffixLen is the maximum length of a suffix generated by this package. -var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", math.MaxInt64)) +var MaxSuffixLen = 1 + len(fmt.Sprintf("%d", int64(math.MaxInt64))) // Comparer is the comparer for test keys generated by this package. -var Comparer *base.Comparer = &base.Comparer{ +var Comparer = &base.Comparer{ Compare: compare, Equal: func(a, b []byte) bool { return compare(a, b) == 0 }, AbbreviatedKey: func(k []byte) uint64 { @@ -136,7 +146,7 @@ func compareTimestamps(a, b []byte) int { // Keyspace describes a finite keyspace of unsuffixed test keys. type Keyspace interface { // Count returns the number of keys that exist within this keyspace. - Count() int + Count() int64 // MaxLen returns the maximum length, in bytes, of a key within this // keyspace. This is only guaranteed to return an upper bound. @@ -144,20 +154,21 @@ type Keyspace interface { // Slice returns the sub-keyspace from index i, inclusive, to index j, // exclusive. The receiver is unmodified. - Slice(i, j int) Keyspace + Slice(i, j int64) Keyspace // EveryN returns a key space that includes 1 key for every N keys in the // original keyspace. The receiver is unmodified. - EveryN(n int) Keyspace + EveryN(n int64) Keyspace - key(buf []byte, i int) int + // key writes the i-th key to the buffer and returns the length. + key(buf []byte, i int64) int } // Divvy divides the provided keyspace into N equal portions, containing // disjoint keys evenly distributed across the keyspace. -func Divvy(ks Keyspace, n int) []Keyspace { +func Divvy(ks Keyspace, n int64) []Keyspace { ret := make([]Keyspace, n) - for i := 0; i < n; i++ { + for i := int64(0); i < n; i++ { ret[i] = ks.Slice(i, ks.Count()).EveryN(n) } return ret @@ -175,7 +186,7 @@ func Alpha(maxLength int) Keyspace { // KeyAt returns the i-th key within the keyspace with a suffix encoding the // timestamp t. -func KeyAt(k Keyspace, i int, t int) []byte { +func KeyAt(k Keyspace, i int64, t int64) []byte { b := make([]byte, k.MaxLen()+MaxSuffixLen) return b[:WriteKeyAt(b, k, i, t)] } @@ -183,20 +194,20 @@ func KeyAt(k Keyspace, i int, t int) []byte { // WriteKeyAt writes the i-th key within the keyspace to the buffer dst, with a // suffix encoding the timestamp t suffix. It returns the number of bytes // written. -func WriteKeyAt(dst []byte, k Keyspace, i int, t int) int { +func WriteKeyAt(dst []byte, k Keyspace, i int64, t int64) int { n := WriteKey(dst, k, i) n += WriteSuffix(dst[n:], t) return n } // Suffix returns the test keys suffix representation of timestamp t. -func Suffix(t int) []byte { +func Suffix(t int64) []byte { b := make([]byte, MaxSuffixLen) return b[:WriteSuffix(b, t)] } // SuffixLen returns the exact length of the given suffix when encoded. -func SuffixLen(t int) int { +func SuffixLen(t int64) int { // Begin at 1 for the '@' delimiter, 1 for a single digit. n := 2 t /= 10 @@ -208,42 +219,42 @@ func SuffixLen(t int) int { } // ParseSuffix returns the integer representation of the encoded suffix. -func ParseSuffix(s []byte) (int, error) { - return strconv.Atoi(strings.TrimPrefix(string(s), string(suffixDelim))) +func ParseSuffix(s []byte) (int64, error) { + return strconv.ParseInt(strings.TrimPrefix(string(s), string(suffixDelim)), 10, 64) } // WriteSuffix writes the test keys suffix representation of timestamp t to dst, // returning the number of bytes written. -func WriteSuffix(dst []byte, t int) int { +func WriteSuffix(dst []byte, t int64) int { dst[0] = suffixDelim n := 1 - n += len(strconv.AppendInt(dst[n:n], int64(t), 10)) + n += len(strconv.AppendInt(dst[n:n], t, 10)) return n } // Key returns the i-th unsuffixed key within the keyspace. -func Key(k Keyspace, i int) []byte { +func Key(k Keyspace, i int64) []byte { b := make([]byte, k.MaxLen()) return b[:k.key(b, i)] } // WriteKey writes the i-th unsuffixed key within the keyspace to the buffer dst. It // returns the number of bytes written. -func WriteKey(dst []byte, k Keyspace, i int) int { +func WriteKey(dst []byte, k Keyspace, i int64) int { return k.key(dst, i) } type alphabet struct { alphabet []byte maxLength int - headSkip int - tailSkip int - increment int + headSkip int64 + tailSkip int64 + increment int64 } -func (a alphabet) Count() int { +func (a alphabet) Count() int64 { // Calculate the total number of keys, ignoring the increment. - total := (keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip) + total := keyCount(len(a.alphabet), a.maxLength) - a.headSkip - a.tailSkip // The increment dictates that we take every N keys, where N = a.increment. // Consider a total containing the 5 keys: @@ -263,20 +274,25 @@ func (a alphabet) MaxLen() int { return a.maxLength } -func (a alphabet) Slice(i, j int) Keyspace { +func (a alphabet) Slice(i, j int64) Keyspace { s := a s.headSkip += i s.tailSkip += a.Count() - j return s } -func (a alphabet) EveryN(n int) Keyspace { +func (a alphabet) EveryN(n int64) Keyspace { s := a s.increment *= n return s } -func keyCount(n, l int) int { +func keyCount(n, l int) int64 { + if n == 0 { + return 0 + } else if n == 1 { + return int64(l) + } // The number of representable keys in the keyspace is a function of the // length of the alphabet n and the max key length l. Consider how the // number of representable keys grows as l increases: @@ -286,10 +302,10 @@ func keyCount(n, l int) int { // l = 3: n + n^2 + n^3 // ... // Σ i=(1...l) n^i = n*(n^l - 1)/(n-1) - return (n * (int(math.Pow(float64(n), float64(l))) - 1)) / (n - 1) + return (int64(n) * (int64(math.Pow(float64(n), float64(l))) - 1)) / int64(n-1) } -func (a alphabet) key(buf []byte, idx int) int { +func (a alphabet) key(buf []byte, idx int64) int { // This function generates keys of length 1..maxKeyLength, pulling // characters from the alphabet. The idx determines which key to generate, // generating the i-th lexicographically next key. @@ -308,14 +324,14 @@ func (a alphabet) key(buf []byte, idx int) int { keyCount(len(a.alphabet), a.maxLength)) } -func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int { +func generateAlphabetKey(buf, alphabet []byte, i, keyCount int64) int { if keyCount == 0 || i > keyCount || i < 0 { return 0 } // Of the keyCount keys in the generative keyspace, how many are there // starting with a particular character? - keysPerCharacter := keyCount / len(alphabet) + keysPerCharacter := keyCount / int64(len(alphabet)) // Find the character that the key at index i starts with and set it. characterIdx := i / keysPerCharacter @@ -347,3 +363,156 @@ func generateAlphabetKey(buf, alphabet []byte, i, keyCount int) int { i = i - keysPerCharacter*characterIdx - 1 return 1 + generateAlphabetKey(buf[1:], alphabet, i, keysPerCharacter-1) } + +// computeAlphabetKeyIndex computes the inverse of generateAlphabetKey, +// returning the index of a particular key, given the provided alphabet and max +// length of a key. +// +// len(key) must be ≥ 1. +func computeAlphabetKeyIndex(key []byte, alphabet map[byte]int64, n int) int64 { + i, ok := alphabet[key[0]] + if !ok { + panic(fmt.Sprintf("unrecognized alphabet character %v", key[0])) + } + // How many keys exist that start with the preceding i characters? Each of + // the i characters themselves are a key, plus the count of all the keys + // with one less character for each. + ret := i + i*keyCount(len(alphabet), n-1) + if len(key) > 1 { + ret += 1 + computeAlphabetKeyIndex(key[1:], alphabet, n-1) + } + return ret +} + +func abs(a int64) int64 { + if a < 0 { + return -a + } + return a +} + +// RandomSeparator returns a random alphabetic key k such that a < k < b, +// pulling randomness from the provided random number generator. If dst is +// provided and the generated key fits within dst's capacity, the returned slice +// will use dst's memory. +// +// If a prefix P exists such that Prefix(a) < P < Prefix(b), the generated key +// will consist of the prefix P appended with the provided suffix. A zero suffix +// generates an unsuffixed key. If no such prefix P exists, RandomSeparator will +// try to find a key k with either Prefix(a) or Prefix(b) such that a < k < b, +// but the generated key will not use the provided suffix. Note that it's +// possible that no separator key exists (eg, a='a@2', b='a@1'), in which case +// RandomSeparator returns nil. +// +// If RandomSeparator generates a new prefix, the generated prefix will have +// length at most MAX(maxLength, len(Prefix(a)), len(Prefix(b))). +// +// RandomSeparator panics if a or b fails to decode. +func RandomSeparator(dst, a, b []byte, suffix int64, maxLength int, rng *rand.Rand) []byte { + if Comparer.Compare(a, b) >= 0 { + return nil + } + + // Determine both keys' logical prefixes and suffixes. + ai := Comparer.Split(a) + bi := Comparer.Split(b) + ap := a[:ai] + bp := b[:bi] + maxLength = max[int](maxLength, max[int](len(ap), len(bp))) + var as, bs int64 + var err error + if ai != len(a) { + as, err = ParseSuffix(a[ai:]) + if err != nil { + panic(fmt.Sprintf("failed to parse suffix of %q", a)) + } + } + if bi != len(b) { + bs, err = ParseSuffix(b[bi:]) + if err != nil { + panic(fmt.Sprintf("failed to parse suffix of %q", b)) + } + } + + apIdx := computeAlphabetKeyIndex(ap, inverseAlphabet, maxLength) + bpIdx := computeAlphabetKeyIndex(bp, inverseAlphabet, maxLength) + diff := bpIdx - apIdx + generatedIdx := bpIdx + if diff > 0 { + var add int64 = diff + 1 + var start int64 = apIdx + if as == 1 { + // There's no expressible key with prefix a greater than a@1. So, + // exclude ap. + start = apIdx + 1 + add = diff + } + if bs == 0 { + // No key with prefix b can sort before b@0. We don't want to pick b. + add-- + } + // We're allowing generated id to be in the range [start, start + add - 1]. + if start > start+add-1 { + return nil + } + // If we can generate a key which is actually in the middle of apIdx + // and bpIdx use it so that we don't have to bother about timestamps. + generatedIdx = rng.Int63n(add) + start + for diff > 1 && generatedIdx == apIdx || generatedIdx == bpIdx { + generatedIdx = rng.Int63n(add) + start + } + } + + switch { + case generatedIdx == apIdx && generatedIdx == bpIdx: + if abs(bs-as) <= 1 { + // There's no expressible suffix between the two, and there's no + // possible separator key. + return nil + } + // The key b is >= key a, but has the same prefix, so b must have the + // smaller timestamp, unless a has timestamp of 0. + // + // NB: The zero suffix (suffix-less) sorts before all other suffixes, so + // any suffix we generate will be greater than it. + if as == 0 { + // bs > as + suffix = bs + rng.Int63n(10) + 1 + } else { + // bs < as. + // Generate suffix in range [bs + 1, as - 1] + suffix = bs + 1 + rng.Int63n(as-bs-1) + } + case generatedIdx == apIdx: + // NB: The zero suffix (suffix-less) sorts before all other suffixes, so + // any suffix we generate will be greater than it. + if as == 0 && suffix == 0 { + suffix++ + } else if as != 0 && suffix >= as { + suffix = rng.Int63n(as) + } + case generatedIdx == bpIdx: + if suffix <= bs { + suffix = bs + rng.Int63n(10) + 1 + } + } + if sz := maxLength + SuffixLen(suffix); cap(dst) < sz { + dst = make([]byte, sz) + } else { + dst = dst[:cap(dst)] + } + var w int + if suffix == 0 { + w = WriteKey(dst, Alpha(maxLength), generatedIdx) + } else { + w = WriteKeyAt(dst, Alpha(maxLength), generatedIdx, suffix) + } + return dst[:w] +} + +func max[I constraints.Ordered](a, b I) I { + if b > a { + return b + } + return a +} diff --git a/vendor/github.com/cockroachdb/pebble/internal_iterator.go b/vendor/github.com/cockroachdb/pebble/internal_iterator.go deleted file mode 100644 index f399a18..0000000 --- a/vendor/github.com/cockroachdb/pebble/internal_iterator.go +++ /dev/null @@ -1,332 +0,0 @@ -// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -package pebble - -import ( - "context" - - "github.com/cockroachdb/pebble/internal/base" - "github.com/cockroachdb/pebble/internal/keyspan" - "github.com/cockroachdb/pebble/internal/manifest" -) - -// scanInternalIterator is an iterator that returns all internal keys instead of -// collapsing them by user keys. For instance, an InternalKeyKindDelete would be -// returned as an InternalKeyKindDelete instead of the iterator skipping over to -// the next key. Useful if an external user of Pebble needs to observe and -// rebuild Pebble's history of internal keys, such as in node-to-node -// replication. For use with {db,snapshot}.ScanInternal(). -// -// scanInternalIterator is allowed to ignorepoint keys deleted by range deletions, -// and range keys shadowed by a range key unset or delete, however it *must* -// return the range delete as well as the range key unset/delete that did the -// shadowing. -type scanInternalIterator struct { - opts IterOptions - comparer *base.Comparer - iter internalIterator - readState *readState - rangeKey *iteratorRangeKeyState - pointKeyIter keyspan.InterleavingIter - iterKey *InternalKey - iterValue LazyValue - alloc *iterAlloc - newIters tableNewIters - newIterRangeKey keyspan.TableNewSpanIter - seqNum uint64 - - // boundsBuf holds two buffers used to store the lower and upper bounds. - // Whenever the InternalIterator's bounds change, the new bounds are copied - // into boundsBuf[boundsBufIdx]. The two bounds share a slice to reduce - // allocations. opts.LowerBound and opts.UpperBound point into this slice. - boundsBuf [2][]byte - boundsBufIdx int -} - -// TODO(sumeer): scanInternalImpl is the implementation for the user-facing -// ScanInternal. Make it accept a context parameter, and plumb it through for -// tracing support. - -func scanInternalImpl( - lower []byte, - iter *scanInternalIterator, - visitPointKey func(key *InternalKey, value LazyValue) error, - visitRangeDel func(start, end []byte, seqNum uint64) error, - visitRangeKey func(start, end []byte, keys []keyspan.Key) error, -) error { - for valid := iter.seekGE(lower); valid && iter.error() == nil; valid = iter.next() { - key := iter.unsafeKey() - - switch key.Kind() { - case InternalKeyKindRangeKeyDelete, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet: - span := iter.unsafeSpan() - if err := visitRangeKey(span.Start, span.End, span.Keys); err != nil { - return err - } - case InternalKeyKindRangeDelete: - rangeDel := iter.unsafeRangeDel() - if err := visitRangeDel(rangeDel.Start, rangeDel.End, rangeDel.LargestSeqNum()); err != nil { - return err - } - default: - val := iter.lazyValue() - if s := iter.pointKeyIter.Span(); s != nil && s.CoversAt(iter.seqNum, key.SeqNum()) { - // Key deleted by a range deletion. Skip it. - continue - } - if err := visitPointKey(key, val); err != nil { - return err - } - } - } - - return nil -} - -// constructPointIter constructs a merging iterator and sets i.iter to it. -func (i *scanInternalIterator) constructPointIter(memtables flushableList, buf *iterAlloc) { - // Merging levels and levels from iterAlloc. - mlevels := buf.mlevels[:0] - levels := buf.levels[:0] - - // We compute the number of levels needed ahead of time and reallocate a slice if - // the array from the iterAlloc isn't large enough. Doing this allocation once - // should improve the performance. - numMergingLevels := len(memtables) - numLevelIters := 0 - - current := i.readState.current - numMergingLevels += len(current.L0SublevelFiles) - numLevelIters += len(current.L0SublevelFiles) - for level := 1; level < len(current.Levels); level++ { - if current.Levels[level].Empty() { - continue - } - numMergingLevels++ - numLevelIters++ - } - - if numMergingLevels > cap(mlevels) { - mlevels = make([]mergingIterLevel, 0, numMergingLevels) - } - if numLevelIters > cap(levels) { - levels = make([]levelIter, 0, numLevelIters) - } - // TODO(bilal): Push these into the iterAlloc buf. - var rangeDelMiter keyspan.MergingIter - rangeDelIters := make([]keyspan.FragmentIterator, 0, numMergingLevels) - rangeDelLevels := make([]keyspan.LevelIter, 0, numLevelIters) - - // Next are the memtables. - for j := len(memtables) - 1; j >= 0; j-- { - mem := memtables[j] - mlevels = append(mlevels, mergingIterLevel{ - iter: mem.newIter(&i.opts), - }) - if rdi := mem.newRangeDelIter(&i.opts); rdi != nil { - rangeDelIters = append(rangeDelIters, rdi) - } - } - - // Next are the file levels: L0 sub-levels followed by lower levels. - mlevelsIndex := len(mlevels) - levelsIndex := len(levels) - mlevels = mlevels[:numMergingLevels] - levels = levels[:numLevelIters] - rangeDelLevels = rangeDelLevels[:numLevelIters] - addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Level) { - li := &levels[levelsIndex] - rli := &rangeDelLevels[levelsIndex] - - li.init( - context.Background(), i.opts, i.comparer.Compare, i.comparer.Split, i.newIters, files, level, - internalIterOpts{}) - li.initBoundaryContext(&mlevels[mlevelsIndex].levelIterBoundaryContext) - mlevels[mlevelsIndex].iter = li - rli.Init(keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}, - i.comparer.Compare, tableNewRangeDelIter(context.Background(), i.newIters), files, level, - manifest.KeyTypePoint) - rangeDelIters = append(rangeDelIters, rli) - - levelsIndex++ - mlevelsIndex++ - } - - // Add level iterators for the L0 sublevels, iterating from newest to - // oldest. - for i := len(current.L0SublevelFiles) - 1; i >= 0; i-- { - addLevelIterForFiles(current.L0SublevelFiles[i].Iter(), manifest.L0Sublevel(i)) - } - - // Add level iterators for the non-empty non-L0 levels. - for level := 1; level < numLevels; level++ { - if current.Levels[level].Empty() { - continue - } - addLevelIterForFiles(current.Levels[level].Iter(), manifest.Level(level)) - } - buf.merging.init(&i.opts, &InternalIteratorStats{}, i.comparer.Compare, i.comparer.Split, mlevels...) - buf.merging.snapshot = i.seqNum - rangeDelMiter.Init(i.comparer.Compare, keyspan.VisibleTransform(i.seqNum), new(keyspan.MergingBuffers), rangeDelIters...) - i.pointKeyIter.Init(i.comparer, &buf.merging, &rangeDelMiter, nil /* mask */, i.opts.LowerBound, i.opts.UpperBound) - i.iter = &i.pointKeyIter -} - -// constructRangeKeyIter constructs the range-key iterator stack, populating -// i.rangeKey.rangeKeyIter with the resulting iterator. This is similar to -// Iterator.constructRangeKeyIter, except it doesn't handle batches and ensures -// iterConfig does *not* elide unsets/deletes. -func (i *scanInternalIterator) constructRangeKeyIter() { - // We want the bounded iter from iterConfig, but not the collapsing of - // RangeKeyUnsets and RangeKeyDels. - i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init( - i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound, - nil /* hasPrefix */, nil /* prefix */, false, /* onlySets */ - &i.rangeKey.rangeKeyBuffers.internal) - - // Next are the flushables: memtables and large batches. - for j := len(i.readState.memtables) - 1; j >= 0; j-- { - mem := i.readState.memtables[j] - // We only need to read from memtables which contain sequence numbers older - // than seqNum. - if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum { - continue - } - if rki := mem.newRangeKeyIter(&i.opts); rki != nil { - i.rangeKey.iterConfig.AddLevel(rki) - } - } - - current := i.readState.current - // Next are the file levels: L0 sub-levels followed by lower levels. - // - // Add file-specific iterators for L0 files containing range keys. This is less - // efficient than using levelIters for sublevels of L0 files containing - // range keys, but range keys are expected to be sparse anyway, reducing the - // cost benefit of maintaining a separate L0Sublevels instance for range key - // files and then using it here. - // - // NB: We iterate L0's files in reverse order. They're sorted by - // LargestSeqNum ascending, and we need to add them to the merging iterator - // in LargestSeqNum descending to preserve the merging iterator's invariants - // around Key Trailer order. - iter := current.RangeKeyLevels[0].Iter() - for f := iter.Last(); f != nil; f = iter.Prev() { - spanIterOpts := &keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters} - spanIter, err := i.newIterRangeKey(f, spanIterOpts) - if err != nil { - i.rangeKey.iterConfig.AddLevel(&errorKeyspanIter{err: err}) - continue - } - i.rangeKey.iterConfig.AddLevel(spanIter) - } - - // Add level iterators for the non-empty non-L0 levels. - for level := 1; level < len(current.RangeKeyLevels); level++ { - if current.RangeKeyLevels[level].Empty() { - continue - } - li := i.rangeKey.iterConfig.NewLevelIter() - spanIterOpts := keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters} - li.Init(spanIterOpts, i.comparer.Compare, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(), - manifest.Level(level), manifest.KeyTypeRange) - i.rangeKey.iterConfig.AddLevel(li) - } -} - -// seekGE seeks this iterator to the first key that's greater than or equal -// to the specified user key. -func (i *scanInternalIterator) seekGE(key []byte) bool { - i.iterKey, i.iterValue = i.iter.SeekGE(key, base.SeekGEFlagsNone) - return i.iterKey != nil -} - -// unsafeKey returns the unsafe InternalKey at the current position. The value -// is nil if the iterator is invalid or exhausted. -func (i *scanInternalIterator) unsafeKey() *InternalKey { - return i.iterKey -} - -// lazyValue returns a value pointer to the value at the current iterator -// position. Behaviour undefined if unsafeKey() returns a Range key or Rangedel -// kind key. -func (i *scanInternalIterator) lazyValue() LazyValue { - return i.iterValue -} - -// unsafeRangeDel returns a range key span. Behaviour undefined if UnsafeKey returns -// a non-rangedel kind. -func (i *scanInternalIterator) unsafeRangeDel() *keyspan.Span { - return i.pointKeyIter.Span() -} - -// unsafeSpan returns a range key span. Behaviour undefined if UnsafeKey returns -// a non-rangekey type. -func (i *scanInternalIterator) unsafeSpan() *keyspan.Span { - return i.rangeKey.iiter.Span() -} - -// next advances the iterator in the forward direction, and returns the -// iterator's new validity state. -func (i *scanInternalIterator) next() bool { - i.iterKey, i.iterValue = i.iter.Next() - return i.iterKey != nil -} - -// error returns an error from the internal iterator, if there's any. -func (i *scanInternalIterator) error() error { - return i.iter.Error() -} - -// close closes this iterator, and releases any pooled objects. -func (i *scanInternalIterator) close() error { - if err := i.iter.Close(); err != nil { - return err - } - i.readState.unref() - if i.rangeKey != nil { - i.rangeKey.PrepareForReuse() - *i.rangeKey = iteratorRangeKeyState{ - rangeKeyBuffers: i.rangeKey.rangeKeyBuffers, - } - iterRangeKeyStateAllocPool.Put(i.rangeKey) - i.rangeKey = nil - } - if alloc := i.alloc; alloc != nil { - for j := range i.boundsBuf { - if cap(i.boundsBuf[j]) >= maxKeyBufCacheSize { - alloc.boundsBuf[j] = nil - } else { - alloc.boundsBuf[j] = i.boundsBuf[j] - } - } - *alloc = iterAlloc{ - keyBuf: alloc.keyBuf[:0], - boundsBuf: alloc.boundsBuf, - prefixOrFullSeekKey: alloc.prefixOrFullSeekKey[:0], - } - iterAllocPool.Put(alloc) - i.alloc = nil - } - return nil -} - -func (i *scanInternalIterator) initializeBoundBufs(lower, upper []byte) { - buf := i.boundsBuf[i.boundsBufIdx][:0] - if lower != nil { - buf = append(buf, lower...) - i.opts.LowerBound = buf - } else { - i.opts.LowerBound = nil - } - if upper != nil { - buf = append(buf, upper...) - i.opts.UpperBound = buf[len(buf)-len(upper):] - } else { - i.opts.UpperBound = nil - } - i.boundsBuf[i.boundsBufIdx] = buf - i.boundsBufIdx = 1 - i.boundsBufIdx -} diff --git a/vendor/github.com/cockroachdb/pebble/iterator.go b/vendor/github.com/cockroachdb/pebble/iterator.go index f58588f..2eee5b2 100644 --- a/vendor/github.com/cockroachdb/pebble/iterator.go +++ b/vendor/github.com/cockroachdb/pebble/iterator.go @@ -9,7 +9,6 @@ import ( "context" "io" "sync" - "sync/atomic" "unsafe" "github.com/cockroachdb/errors" @@ -188,7 +187,9 @@ type Iterator struct { comparer base.Comparer iter internalIterator pointIter internalIterator + // Either readState or version is set, but not both. readState *readState + version *version // rangeKey holds iteration state specific to iteration over range keys. // The range key field may be nil if the Iterator has never been configured // to iterate over range keys. Its non-nilness cannot be used to determine @@ -446,6 +447,11 @@ const ( seekPrefixGELastPositioningOp seekGELastPositioningOp seekLTLastPositioningOp + // internalNextOp is a special internal iterator positioning operation used + // by CanDeterministicallySingleDelete. It exists for enforcing requirements + // around calling CanDeterministicallySingleDelete at most once per external + // iterator position. + internalNextOp // invalidatedLastPositionOp is similar to unknownLastPositionOp and the // only reason to distinguish this is for the wider set of SeekGE // optimizations we permit for the external iterator Iterator.forwardOnly @@ -549,6 +555,21 @@ func (i *Iterator) findNextEntry(limit []byte) { return } + // If the user has configured a SkipPoint function, invoke it to see + // whether we should skip over the current user key. + if i.opts.SkipPoint != nil && key.Kind() != InternalKeyKindRangeKeySet && i.opts.SkipPoint(i.iterKey.UserKey) { + // NB: We could call nextUserKey, but in some cases the SkipPoint + // predicate function might be cheaper than nextUserKey's key copy + // and key comparison. This should be the case for MVCC suffix + // comparisons, for example. In the future, we could expand the + // SkipPoint interface to give the implementor more control over + // whether we skip over just the internal key, the user key, or even + // the key prefix. + i.stats.ForwardStepCount[InternalIterCall]++ + i.iterKey, i.iterValue = i.iter.Next() + continue + } + switch key.Kind() { case InternalKeyKindRangeKeySet: // Save the current key. @@ -569,7 +590,10 @@ func (i *Iterator) findNextEntry(limit []byte) { i.iterValidityState = IterValid return - case InternalKeyKindDelete, InternalKeyKindSingleDelete: + case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized: + // NB: treating InternalKeyKindSingleDelete as equivalent to DEL is not + // only simpler, but is also necessary for correctness due to + // InternalKeyKindSSTableInternalObsoleteBit. i.nextUserKey() continue @@ -615,6 +639,13 @@ func (i *Iterator) findNextEntry(limit []byte) { } func (i *Iterator) nextPointCurrentUserKey() bool { + // If the user has configured a SkipPoint function and the current user key + // would be skipped by it, there's no need to step forward looking for a + // point key. If we were to find one, it should be skipped anyways. + if i.opts.SkipPoint != nil && i.opts.SkipPoint(i.key) { + return false + } + i.pos = iterPosCurForward i.iterKey, i.iterValue = i.iter.Next() @@ -632,7 +663,10 @@ func (i *Iterator) nextPointCurrentUserKey() bool { i.err = base.CorruptionErrorf("pebble: unexpected range key set mid-user key") return false - case InternalKeyKindDelete, InternalKeyKindSingleDelete: + case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized: + // NB: treating InternalKeyKindSingleDelete as equivalent to DEL is not + // only simpler, but is also necessary for correctness due to + // InternalKeyKindSSTableInternalObsoleteBit. return false case InternalKeyKindSet, InternalKeyKindSetWithDelete: @@ -773,52 +807,54 @@ func (i *Iterator) maybeSampleRead() { func (i *Iterator) sampleRead() { var topFile *manifest.FileMetadata topLevel, numOverlappingLevels := numLevels, 0 - if mi, ok := i.iter.(*mergingIter); ok { - if len(mi.levels) > 1 { - mi.ForEachLevelIter(func(li *levelIter) bool { - l := manifest.LevelToInt(li.level) - if f := li.iterFile; f != nil { - var containsKey bool - if i.pos == iterPosNext || i.pos == iterPosCurForward || - i.pos == iterPosCurForwardPaused { - containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0 - } else if i.pos == iterPosPrev || i.pos == iterPosCurReverse || - i.pos == iterPosCurReversePaused { - containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0 - } - // Do nothing if the current key is not contained in f's - // bounds. We could seek the LevelIterator at this level - // to find the right file, but the performance impacts of - // doing that are significant enough to negate the benefits - // of read sampling in the first place. See the discussion - // at: - // https://github.com/cockroachdb/pebble/pull/1041#issuecomment-763226492 - if containsKey { - numOverlappingLevels++ - if numOverlappingLevels >= 2 { - // Terminate the loop early if at least 2 overlapping levels are found. - return true - } - topLevel = l - topFile = f - } + mi := i.merging + if mi == nil { + return + } + if len(mi.levels) > 1 { + mi.ForEachLevelIter(func(li *levelIter) bool { + l := manifest.LevelToInt(li.level) + if f := li.iterFile; f != nil { + var containsKey bool + if i.pos == iterPosNext || i.pos == iterPosCurForward || + i.pos == iterPosCurForwardPaused { + containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0 + } else if i.pos == iterPosPrev || i.pos == iterPosCurReverse || + i.pos == iterPosCurReversePaused { + containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0 } - return false - }) - } + // Do nothing if the current key is not contained in f's + // bounds. We could seek the LevelIterator at this level + // to find the right file, but the performance impacts of + // doing that are significant enough to negate the benefits + // of read sampling in the first place. See the discussion + // at: + // https://github.com/cockroachdb/pebble/pull/1041#issuecomment-763226492 + if containsKey { + numOverlappingLevels++ + if numOverlappingLevels >= 2 { + // Terminate the loop early if at least 2 overlapping levels are found. + return true + } + topLevel = l + topFile = f + } + } + return false + }) } if topFile == nil || topLevel >= numLevels { return } if numOverlappingLevels >= 2 { - allowedSeeks := atomic.AddInt64(&topFile.Atomic.AllowedSeeks, -1) + allowedSeeks := topFile.AllowedSeeks.Add(-1) if allowedSeeks == 0 { // Since the compaction queue can handle duplicates, we can keep // adding to the queue even once allowedSeeks hits 0. // In fact, we NEED to keep adding to the queue, because the queue // is small and evicts older and possibly useful compactions. - atomic.AddInt64(&topFile.Atomic.AllowedSeeks, topFile.InitAllowedSeeks) + topFile.AllowedSeeks.Add(topFile.InitAllowedSeeks) read := readCompaction{ start: topFile.SmallestPointKey.UserKey, @@ -902,6 +938,26 @@ func (i *Iterator) findPrevEntry(limit []byte) { } } + // If the user has configured a SkipPoint function, invoke it to see + // whether we should skip over the current user key. + if i.opts.SkipPoint != nil && key.Kind() != InternalKeyKindRangeKeySet && i.opts.SkipPoint(key.UserKey) { + // NB: We could call prevUserKey, but in some cases the SkipPoint + // predicate function might be cheaper than prevUserKey's key copy + // and key comparison. This should be the case for MVCC suffix + // comparisons, for example. In the future, we could expand the + // SkipPoint interface to give the implementor more control over + // whether we skip over just the internal key, the user key, or even + // the key prefix. + i.stats.ReverseStepCount[InternalIterCall]++ + i.iterKey, i.iterValue = i.iter.Prev() + if limit != nil && i.iterKey != nil && i.cmp(limit, i.iterKey.UserKey) > 0 && !i.rangeKeyWithinLimit(limit) { + i.iterValidityState = IterAtLimit + i.pos = iterPosCurReversePaused + return + } + continue + } + switch key.Kind() { case InternalKeyKindRangeKeySet: // Range key start boundary markers are interleaved with the maximum @@ -930,7 +986,7 @@ func (i *Iterator) findPrevEntry(limit []byte) { // return the key even if the MERGE point key is deleted. rangeKeyBoundary = true - case InternalKeyKindDelete, InternalKeyKindSingleDelete: + case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized: i.value = LazyValue{} i.iterValidityState = IterExhausted valueMerger = nil @@ -939,12 +995,12 @@ func (i *Iterator) findPrevEntry(limit []byte) { // Compare with the limit. We could optimize by only checking when // we step to the previous user key, but detecting that requires a // comparison too. Note that this position may already passed a - // number of versions of this user key, but they are all deleted, - // so the fact that a subsequent Prev*() call will not see them is + // number of versions of this user key, but they are all deleted, so + // the fact that a subsequent Prev*() call will not see them is // harmless. Also note that this is the only place in the loop, - // other than the firstLoopIter case above, where we could step - // to a different user key and start processing it for returning - // to the caller. + // other than the firstLoopIter and SkipPoint cases above, where we + // could step to a different user key and start processing it for + // returning to the caller. if limit != nil && i.iterKey != nil && i.cmp(limit, i.iterKey.UserKey) > 0 && !i.rangeKeyWithinLimit(limit) { i.iterValidityState = IterAtLimit i.pos = iterPosCurReversePaused @@ -1093,9 +1149,13 @@ func (i *Iterator) mergeNext(key InternalKey, valueMerger ValueMerger) { return } switch key.Kind() { - case InternalKeyKindDelete, InternalKeyKindSingleDelete: + case InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized: // We've hit a deletion tombstone. Return everything up to this // point. + // + // NB: treating InternalKeyKindSingleDelete as equivalent to DEL is not + // only simpler, but is also necessary for correctness due to + // InternalKeyKindSSTableInternalObsoleteBit. return case InternalKeyKindSet, InternalKeyKindSetWithDelete: @@ -1751,7 +1811,8 @@ func (i *Iterator) internalNextPrefix(currKeyPrefixLen int) { i.iterKey, i.iterValue = i.iter.NextPrefix(i.prefixOrFullSeekKey) if invariants.Enabled && i.iterKey != nil { if iterKeyPrefixLen := i.split(i.iterKey.UserKey); i.cmp(i.iterKey.UserKey[:iterKeyPrefixLen], i.prefixOrFullSeekKey) < 0 { - panic("pebble: iter.NextPrefix did not advance beyond the current prefix") + panic(errors.AssertionFailedf("pebble: iter.NextPrefix did not advance beyond the current prefix: now at %q; expected to be geq %q", + i.iterKey, i.prefixOrFullSeekKey)) } } } @@ -2120,6 +2181,9 @@ func (i *Iterator) Value() []byte { // ValueAndErr returns the value, and any error encountered in extracting the value. // REQUIRES: i.Error()==nil and HasPointAndRange() returns true for hasPoint. +// +// The caller should not modify the contents of the returned slice, and its +// contents may change on the next call to Next. func (i *Iterator) ValueAndErr() ([]byte, error) { val, callerOwned, err := i.value.Value(i.lazyValueBuf) if err != nil { @@ -2150,7 +2214,13 @@ func (i *Iterator) RangeKeys() []RangeKeyData { // Valid returns true if the iterator is positioned at a valid key/value pair // and false otherwise. func (i *Iterator) Valid() bool { - return i.iterValidityState == IterValid && !i.requiresReposition + valid := i.iterValidityState == IterValid && !i.requiresReposition + if invariants.Enabled { + if err := i.Error(); valid && err != nil { + panic(errors.WithSecondaryError(errors.AssertionFailedf("pebble: iterator is valid with non-nil Error"), err)) + } + } + return valid } // Error returns any accumulated error. @@ -2215,6 +2285,10 @@ func (i *Iterator) Close() error { i.readState = nil } + if i.version != nil { + i.version.Unref() + } + for _, readers := range i.externalReaders { for _, r := range readers { err = firstError(err, r.Close()) @@ -2407,7 +2481,8 @@ func (i *Iterator) SetOptions(o *IterOptions) { // If either options specify block property filters for an iterator stack, // reconstruct it. if i.pointIter != nil && (closeBoth || len(o.PointKeyFilters) > 0 || len(i.opts.PointKeyFilters) > 0 || - o.RangeKeyMasking.Filter != nil || i.opts.RangeKeyMasking.Filter != nil) { + o.RangeKeyMasking.Filter != nil || i.opts.RangeKeyMasking.Filter != nil || o.SkipPoint != nil || + i.opts.SkipPoint != nil) { i.err = firstError(i.err, i.pointIter.Close()) i.pointIter = nil } @@ -2646,11 +2721,22 @@ func (i *Iterator) CloneWithContext(ctx context.Context, opts CloneOptions) (*It } readState := i.readState - if readState == nil { + vers := i.version + if readState == nil && vers == nil { return nil, errors.Errorf("cannot Clone a closed Iterator") } // i is already holding a ref, so there is no race with unref here. - readState.ref() + // + // TODO(bilal): If the underlying iterator was created on a snapshot, we could + // grab a reference to the current readState instead of reffing the original + // readState. This allows us to release references to some zombie sstables + // and memtables. + if readState != nil { + readState.ref() + } + if vers != nil { + vers.Ref() + } // Bundle various structures under a single umbrella in order to allocate // them together. buf := iterAllocPool.Get().(*iterAlloc) @@ -2662,6 +2748,7 @@ func (i *Iterator) CloneWithContext(ctx context.Context, opts CloneOptions) (*It merge: i.merge, comparer: i.comparer, readState: readState, + version: vers, keyBuf: buf.keyBuf, prefixOrFullSeekKey: buf.prefixOrFullSeekKey, boundsBuf: buf.boundsBuf, @@ -2716,19 +2803,19 @@ func (stats *IteratorStats) SafeFormat(s redact.SafePrinter, verb rune) { s.SafeString(",\n(internal-stats: ") s.Printf("(block-bytes: (total %s, cached %s, read-time %s)), "+ "(points: (count %s, key-bytes %s, value-bytes %s, tombstoned %s))", - humanize.IEC.Uint64(stats.InternalStats.BlockBytes), - humanize.IEC.Uint64(stats.InternalStats.BlockBytesInCache), + humanize.Bytes.Uint64(stats.InternalStats.BlockBytes), + humanize.Bytes.Uint64(stats.InternalStats.BlockBytesInCache), humanize.FormattedString(stats.InternalStats.BlockReadDuration.String()), - humanize.SI.Uint64(stats.InternalStats.PointCount), - humanize.SI.Uint64(stats.InternalStats.KeyBytes), - humanize.SI.Uint64(stats.InternalStats.ValueBytes), - humanize.SI.Uint64(stats.InternalStats.PointsCoveredByRangeTombstones), + humanize.Count.Uint64(stats.InternalStats.PointCount), + humanize.Bytes.Uint64(stats.InternalStats.KeyBytes), + humanize.Bytes.Uint64(stats.InternalStats.ValueBytes), + humanize.Count.Uint64(stats.InternalStats.PointsCoveredByRangeTombstones), ) if stats.InternalStats.SeparatedPointValue.Count != 0 { s.Printf(", (separated: (count %s, bytes %s, fetched %s)))", - humanize.SI.Uint64(stats.InternalStats.SeparatedPointValue.Count), - humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytes), - humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytesFetched)) + humanize.Count.Uint64(stats.InternalStats.SeparatedPointValue.Count), + humanize.Bytes.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytes), + humanize.Bytes.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytesFetched)) } else { s.Printf(")") } @@ -2741,3 +2828,169 @@ func (stats *IteratorStats) SafeFormat(s redact.SafePrinter, verb rune) { stats.RangeKeyStats.SkippedPoints) } } + +// CanDeterministicallySingleDelete takes a valid iterator and examines internal +// state to determine if a SingleDelete deleting Iterator.Key() would +// deterministically delete the key. CanDeterministicallySingleDelete requires +// the iterator to be oriented in the forward direction (eg, the last +// positioning operation must've been a First, a Seek[Prefix]GE, or a +// Next[Prefix][WithLimit]). +// +// This function does not change the external position of the iterator, and all +// positioning methods should behave the same as if it was never called. This +// function will only return a meaningful result the first time it's invoked at +// an iterator position. This function invalidates the iterator Value's memory, +// and the caller must not rely on the memory safety of the previous Iterator +// position. +// +// If CanDeterministicallySingleDelete returns true AND the key at the iterator +// position is not modified between the creation of the Iterator and the commit +// of a batch containing a SingleDelete over the key, then the caller can be +// assured that SingleDelete is equivalent to Delete on the local engine, but it +// may not be true on another engine that received the same writes and with +// logically equivalent state since this engine may have collapsed multiple SETs +// into one. +func CanDeterministicallySingleDelete(it *Iterator) (bool, error) { + // This function may only be called once per external iterator position. We + // can validate this by checking the last positioning operation. + if it.lastPositioningOp == internalNextOp { + return false, errors.New("pebble: CanDeterministicallySingleDelete called twice") + } + validity, kind := it.internalNext() + var shadowedBySingleDelete bool + for validity == internalNextValid { + switch kind { + case InternalKeyKindDelete, InternalKeyKindDeleteSized: + // A DEL or DELSIZED tombstone is okay. An internal key + // sequence like SINGLEDEL; SET; DEL; SET can be handled + // deterministically. If there are SETs further down, we + // don't care about them. + return true, nil + case InternalKeyKindSingleDelete: + // A SingleDelete is okay as long as when that SingleDelete was + // written, it was written deterministically (eg, with its own + // CanDeterministicallySingleDelete check). Validate that it was + // written deterministically. We'll allow one set to appear after + // the SingleDelete. + shadowedBySingleDelete = true + validity, kind = it.internalNext() + continue + case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge: + // If we observed a single delete, it's allowed to delete 1 key. + // We'll keep looping to validate that the internal keys beneath the + // already-written single delete are copacetic. + if shadowedBySingleDelete { + shadowedBySingleDelete = false + validity, kind = it.internalNext() + continue + } + // We encountered a shadowed SET, SETWITHDEL, MERGE. A SINGLEDEL + // that deleted the KV at the original iterator position could + // result in this key becoming visible. + return false, nil + case InternalKeyKindRangeDelete: + // RangeDeletes are handled by the merging iterator and should never + // be observed by the top-level Iterator. + panic(errors.AssertionFailedf("pebble: unexpected range delete")) + case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: + // Range keys are interleaved at the maximal sequence number and + // should never be observed within a user key. + panic(errors.AssertionFailedf("pebble: unexpected range key")) + default: + panic(errors.AssertionFailedf("pebble: unexpected key kind: %s", errors.Safe(kind))) + } + } + if validity == internalNextError { + return false, it.Error() + } + return true, nil +} + +// internalNextValidity enumerates the potential outcomes of a call to +// internalNext. +type internalNextValidity int8 + +const ( + // internalNextError is returned by internalNext when an error occurred and + // the caller is responsible for checking iter.Error(). + internalNextError internalNextValidity = iota + // internalNextExhausted is returned by internalNext when the next internal + // key is an internal key with a different user key than Iterator.Key(). + internalNextExhausted + // internalNextValid is returned by internalNext when the internal next + // found a shadowed internal key with a user key equal to Iterator.Key(). + internalNextValid +) + +// internalNext advances internal Iterator state forward to expose the +// InternalKeyKind of the next internal key with a user key equal to Key(). +// +// internalNext is a highly specialized operation and is unlikely to be +// generally useful. See Iterator.Next for how to reposition the iterator to the +// next key. internalNext requires the Iterator to be at a valid position in the +// forward direction (the last positioning operation must've been a First, a +// Seek[Prefix]GE, or a Next[Prefix][WithLimit] and Valid() must return true). +// +// internalNext, unlike all other Iterator methods, exposes internal LSM state. +// internalNext advances the Iterator's internal iterator to the next shadowed +// key with a user key equal to Key(). When a key is overwritten or deleted, its +// removal from the LSM occurs lazily as a part of compactions. internalNext +// allows the caller to see whether an obsolete internal key exists with the +// current Key(), and what it's key kind is. Note that the existence of an +// internal key is nondeterministic and dependent on internal LSM state. These +// semantics are unlikely to be applicable to almost all use cases. +// +// If internalNext finds a key that shares the same user key as Key(), it +// returns internalNextValid and the internal key's kind. If internalNext +// encounters an error, it returns internalNextError and the caller is expected +// to call Iterator.Error() to retrieve it. In all other circumstances, +// internalNext returns internalNextExhausted, indicating that there are no more +// additional internal keys with the user key Key(). +// +// internalNext does not change the external position of the iterator, and a +// Next operation should behave the same as if internalNext was never called. +// internalNext does invalidate the iterator Value's memory, and the caller must +// not rely on the memory safety of the previous Iterator position. +func (i *Iterator) internalNext() (internalNextValidity, base.InternalKeyKind) { + i.stats.ForwardStepCount[InterfaceCall]++ + if i.err != nil { + return internalNextError, base.InternalKeyKindInvalid + } else if i.iterValidityState != IterValid { + return internalNextExhausted, base.InternalKeyKindInvalid + } + i.lastPositioningOp = internalNextOp + + switch i.pos { + case iterPosCurForward: + i.iterKey, i.iterValue = i.iter.Next() + if i.iterKey == nil { + // We check i.iter.Error() here and return an internalNextError enum + // variant so that the caller does not need to check i.iter.Error() + // in the common case that the next internal key has a new user key. + if i.err = i.iter.Error(); i.err != nil { + return internalNextError, base.InternalKeyKindInvalid + } + i.pos = iterPosNext + return internalNextExhausted, base.InternalKeyKindInvalid + } else if i.comparer.Equal(i.iterKey.UserKey, i.key) { + return internalNextValid, i.iterKey.Kind() + } + i.pos = iterPosNext + return internalNextExhausted, base.InternalKeyKindInvalid + case iterPosCurReverse, iterPosCurReversePaused, iterPosPrev: + i.err = errors.New("switching from reverse to forward via internalNext is prohibited") + i.iterValidityState = IterExhausted + return internalNextError, base.InternalKeyKindInvalid + case iterPosNext, iterPosCurForwardPaused: + // The previous method already moved onto the next user key. This is + // only possible if + // - the last positioning method was a call to internalNext, and we + // advanced to a new user key. + // - the previous non-internalNext iterator operation encountered a + // range key or merge, forcing an internal Next that found a new + // user key that's not equal to i.Iterator.Key(). + return internalNextExhausted, base.InternalKeyKindInvalid + default: + panic("unreachable") + } +} diff --git a/vendor/github.com/cockroachdb/pebble/level_checker.go b/vendor/github.com/cockroachdb/pebble/level_checker.go index 89f18b0..2901c45 100644 --- a/vendor/github.com/cockroachdb/pebble/level_checker.go +++ b/vendor/github.com/cockroachdb/pebble/level_checker.go @@ -9,7 +9,6 @@ import ( "fmt" "io" "sort" - "sync/atomic" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" @@ -166,7 +165,7 @@ func (m *simpleMergingIter) step() bool { if m.valueMerger != nil { // Ongoing series of MERGE records. switch item.key.Kind() { - case InternalKeyKindSingleDelete, InternalKeyKindDelete: + case InternalKeyKindSingleDelete, InternalKeyKindDelete, InternalKeyKindDeleteSized: var closer io.Closer _, closer, m.err = m.valueMerger.Finish(true /* includesBase */) if m.err == nil && closer != nil { @@ -346,7 +345,7 @@ func iterateAndCheckTombstones( type checkConfig struct { logger Logger - cmp Compare + comparer *Comparer readState *readState newIters tableNewIters seqNum uint64 @@ -355,6 +354,9 @@ type checkConfig struct { formatKey base.FormatKey } +// cmp is shorthand for comparer.Compare. +func (c *checkConfig) cmp(a, b []byte) int { return c.comparer.Compare(a, b) } + func checkRangeTombstones(c *checkConfig) error { var level int var tombstones []tombstoneWithLevel @@ -380,7 +382,7 @@ func checkRangeTombstones(c *checkConfig) error { atomicUnit, _ := expandToAtomicUnit(c.cmp, lf.Slice(), true /* disableIsCompacting */) lower, upper := manifest.KeyRange(c.cmp, atomicUnit.Iter()) iterToClose, iter, err := c.newIters( - context.Background(), lf.FileMetadata, nil, internalIterOpts{}) + context.Background(), lf.FileMetadata, &IterOptions{level: manifest.Level(lsmLevel)}, internalIterOpts{}) if err != nil { return err } @@ -568,11 +570,11 @@ func (d *DB) CheckLevels(stats *CheckLevelsStats) error { // Determine the seqnum to read at after grabbing the read state (current and // memtables) above. - seqNum := atomic.LoadUint64(&d.mu.versions.atomic.visibleSeqNum) + seqNum := d.mu.versions.visibleSeqNum.Load() checkConfig := &checkConfig{ logger: d.opts.Logger, - cmp: d.cmp, + comparer: d.opts.Comparer, readState: readState, newIters: d.newIters, seqNum: seqNum, @@ -640,7 +642,7 @@ func checkLevelsInternal(c *checkConfig) (err error) { manifestIter := current.L0SublevelFiles[sublevel].Iter() iterOpts := IterOptions{logger: c.logger} li := &levelIter{} - li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters, manifestIter, + li.init(context.Background(), iterOpts, c.comparer, c.newIters, manifestIter, manifest.L0Sublevel(sublevel), internalIterOpts{}) li.initRangeDel(&mlevelAlloc[0].rangeDelIter) li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext) @@ -654,7 +656,7 @@ func checkLevelsInternal(c *checkConfig) (err error) { iterOpts := IterOptions{logger: c.logger} li := &levelIter{} - li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters, + li.init(context.Background(), iterOpts, c.comparer, c.newIters, current.Levels[level].Iter(), manifest.Level(level), internalIterOpts{}) li.initRangeDel(&mlevelAlloc[0].rangeDelIter) li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext) diff --git a/vendor/github.com/cockroachdb/pebble/level_iter.go b/vendor/github.com/cockroachdb/pebble/level_iter.go index feb809d..ae6b045 100644 --- a/vendor/github.com/cockroachdb/pebble/level_iter.go +++ b/vendor/github.com/cockroachdb/pebble/level_iter.go @@ -36,9 +36,8 @@ type tableNewIters func( // tableNewRangeDelIter takes a tableNewIters and returns a TableNewSpanIter // for the rangedel iterator returned by tableNewIters. func tableNewRangeDelIter(ctx context.Context, newIters tableNewIters) keyspan.TableNewSpanIter { - return func(file *manifest.FileMetadata, iterOptions *keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) { - iter, rangeDelIter, err := newIters( - ctx, file, &IterOptions{RangeKeyFilters: iterOptions.RangeKeyFilters}, internalIterOpts{}) + return func(file *manifest.FileMetadata, iterOptions keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) { + iter, rangeDelIter, err := newIters(ctx, file, nil, internalIterOpts{}) if iter != nil { _ = iter.Close() } @@ -51,6 +50,7 @@ func tableNewRangeDelIter(ctx context.Context, newIters tableNewIters) keyspan.T type internalIterOpts struct { bytesIterated *uint64 + bufferPool *sstable.BufferPool stats *base.InternalIteratorStats boundLimitedFilter sstable.BoundLimitedBlockPropertyFilter } @@ -82,10 +82,11 @@ type levelIter struct { // short-lived (since they pin sstables), (b) plumbing a context into every // method is very painful, (c) they do not (yet) respect context // cancellation and are only used for tracing. - ctx context.Context - logger Logger - cmp Compare - split Split + ctx context.Context + logger Logger + comparer *Comparer + cmp Compare + split Split // The lower/upper bounds for iteration as specified at creation or the most // recent call to SetBounds. lower []byte @@ -200,6 +201,11 @@ type levelIter struct { // cache when constructing new table iterators. internalOpts internalIterOpts + // Scratch space for the obsolete keys filter, when there are no other block + // property filters specified. See the performance note where + // IterOptions.PointKeyFilters is declared. + filtersBuf [1]BlockPropertyFilter + // Disable invariant checks even if they are otherwise enabled. Used by tests // which construct "impossible" situations (e.g. seeking to a key before the // lower bound). @@ -236,24 +242,22 @@ var _ base.InternalIterator = (*levelIter)(nil) // parameter if the caller is never going to call SeekPrefixGE. func newLevelIter( opts IterOptions, - cmp Compare, - split Split, + comparer *Comparer, newIters tableNewIters, files manifest.LevelIterator, level manifest.Level, - bytesIterated *uint64, + internalOpts internalIterOpts, ) *levelIter { l := &levelIter{} - l.init(context.Background(), opts, cmp, split, newIters, files, level, - internalIterOpts{bytesIterated: bytesIterated}) + l.init(context.Background(), opts, comparer, newIters, files, level, + internalOpts) return l } func (l *levelIter) init( ctx context.Context, opts IterOptions, - cmp Compare, - split Split, + comparer *Comparer, newIters tableNewIters, files manifest.LevelIterator, level manifest.Level, @@ -267,10 +271,15 @@ func (l *levelIter) init( l.upper = opts.UpperBound l.tableOpts.TableFilter = opts.TableFilter l.tableOpts.PointKeyFilters = opts.PointKeyFilters + if len(opts.PointKeyFilters) == 0 { + l.tableOpts.PointKeyFilters = l.filtersBuf[:0:1] + } l.tableOpts.UseL6Filters = opts.UseL6Filters l.tableOpts.level = l.level - l.cmp = cmp - l.split = split + l.tableOpts.snapshotForHideObsoletePoints = opts.snapshotForHideObsoletePoints + l.comparer = comparer + l.cmp = comparer.Compare + l.split = comparer.Split l.iterFile = nil l.newIters = newIters l.files = files @@ -788,7 +797,7 @@ func (l *levelIter) SeekPrefixGE( } return l.verify(l.largestBoundary, base.LazyValue{}) } - // It is possible that we are here because bloom filter matching failed. In + // It is possible that we are here because bloom filter matching failed. In // that case it is likely that all keys matching the prefix are wholly // within the current file and cannot be in the subsequent file. In that // case we don't want to go to the next file, since loading and seeking in @@ -796,7 +805,16 @@ func (l *levelIter) SeekPrefixGE( // next file will defeat the optimization for the next SeekPrefixGE that is // called with flags.TrySeekUsingNext(), since for sparse key spaces it is // likely that the next key will also be contained in the current file. - if n := l.split(l.iterFile.LargestPointKey.UserKey); l.cmp(prefix, l.iterFile.LargestPointKey.UserKey[:n]) < 0 { + var n int + if l.split != nil { + // If the split function is specified, calculate the prefix length accordingly. + n = l.split(l.iterFile.LargestPointKey.UserKey) + } else { + // If the split function is not specified, the entire key is used as the + // prefix. This case can occur when getIter uses SeekPrefixGE. + n = len(l.iterFile.LargestPointKey.UserKey) + } + if l.cmp(prefix, l.iterFile.LargestPointKey.UserKey[:n]) < 0 { return nil, base.LazyValue{} } return l.verify(l.skipEmptyFileForward()) @@ -1066,7 +1084,9 @@ func (l *levelIter) skipEmptyFileForward() (*InternalKey, base.LazyValue) { if *l.rangeDelIterPtr != nil && l.filteredIter != nil && l.filteredIter.MaybeFilteredKeys() { l.largestBoundary = &l.iterFile.Largest - l.boundaryContext.isIgnorableBoundaryKey = true + if l.boundaryContext != nil { + l.boundaryContext.isIgnorableBoundaryKey = true + } return l.largestBoundary, base.LazyValue{} } } @@ -1155,7 +1175,9 @@ func (l *levelIter) skipEmptyFileBackward() (*InternalKey, base.LazyValue) { // the next file. if *l.rangeDelIterPtr != nil && l.filteredIter != nil && l.filteredIter.MaybeFilteredKeys() { l.smallestBoundary = &l.iterFile.Smallest - l.boundaryContext.isIgnorableBoundaryKey = true + if l.boundaryContext != nil { + l.boundaryContext.isIgnorableBoundaryKey = true + } return l.smallestBoundary, base.LazyValue{} } } diff --git a/vendor/github.com/cockroachdb/pebble/log_recycler.go b/vendor/github.com/cockroachdb/pebble/log_recycler.go index 85e64cc..c8c2ff4 100644 --- a/vendor/github.com/cockroachdb/pebble/log_recycler.go +++ b/vendor/github.com/cockroachdb/pebble/log_recycler.go @@ -32,14 +32,14 @@ type logRecycler struct { // the log file should not be deleted (i.e. the log is being recycled), and // false otherwise. func (r *logRecycler) add(logInfo fileInfo) bool { - if logInfo.fileNum < r.minRecycleLogNum { + if logInfo.fileNum.FileNum() < r.minRecycleLogNum { return false } r.mu.Lock() defer r.mu.Unlock() - if logInfo.fileNum <= r.mu.maxLogNum { + if logInfo.fileNum.FileNum() <= r.mu.maxLogNum { // The log file number was already considered for recycling. Don't consider // it again. This avoids a race between adding the same log file for // recycling multiple times, and removing the log file for actual @@ -49,7 +49,7 @@ func (r *logRecycler) add(logInfo fileInfo) bool { // shouldn't be deleted. return true } - r.mu.maxLogNum = logInfo.fileNum + r.mu.maxLogNum = logInfo.fileNum.FileNum() if len(r.mu.logs) >= r.limit { return false } @@ -89,7 +89,7 @@ func (r *logRecycler) pop(logNum FileNum) error { if len(r.mu.logs) == 0 { return errors.New("pebble: log recycler empty") } - if r.mu.logs[0].fileNum != logNum { + if r.mu.logs[0].fileNum.FileNum() != logNum { return errors.Errorf("pebble: log recycler invalid %d vs %d", errors.Safe(logNum), errors.Safe(fileInfoNums(r.mu.logs))) } r.mu.logs = r.mu.logs[1:] @@ -102,7 +102,7 @@ func fileInfoNums(finfos []fileInfo) []FileNum { } nums := make([]FileNum, len(finfos)) for i := range finfos { - nums[i] = finfos[i].fileNum + nums[i] = finfos[i].fileNum.FileNum() } return nums } diff --git a/vendor/github.com/cockroachdb/pebble/mem_table.go b/vendor/github.com/cockroachdb/pebble/mem_table.go index 5691102..e728e94 100644 --- a/vendor/github.com/cockroachdb/pebble/mem_table.go +++ b/vendor/github.com/cockroachdb/pebble/mem_table.go @@ -10,12 +10,12 @@ import ( "os" "sync" "sync/atomic" - "unsafe" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/arenaskl" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/keyspan" + "github.com/cockroachdb/pebble/internal/manual" "github.com/cockroachdb/pebble/internal/rangedel" "github.com/cockroachdb/pebble/internal/rangekey" ) @@ -80,12 +80,21 @@ type memTable struct { // inflight mutations that have reserved space in the memtable but not yet // applied. The memtable cannot be flushed to disk until the writer refs // drops to zero. - writerRefs int32 + writerRefs atomic.Int32 tombstones keySpanCache rangeKeys keySpanCache // The current logSeqNum at the time the memtable was created. This is // guaranteed to be less than or equal to any seqnum stored in the memtable. - logSeqNum uint64 + logSeqNum uint64 + releaseAccountingReservation func() +} + +func (m *memTable) free() { + if m != nil { + m.releaseAccountingReservation() + manual.Free(m.arenaBuf) + m.arenaBuf = nil + } } // memTableOptions holds configuration used when creating a memTable. All of @@ -93,9 +102,10 @@ type memTable struct { // which is used by tests. type memTableOptions struct { *Options - arenaBuf []byte - size int - logSeqNum uint64 + arenaBuf []byte + size int + logSeqNum uint64 + releaseAccountingReservation func() } func checkMemTable(obj interface{}) { @@ -110,18 +120,24 @@ func checkMemTable(obj interface{}) { // Options.MemTableSize is used instead. func newMemTable(opts memTableOptions) *memTable { opts.Options = opts.Options.EnsureDefaults() - if opts.size == 0 { - opts.size = opts.MemTableSize - } + m := new(memTable) + m.init(opts) + return m +} - m := &memTable{ - cmp: opts.Comparer.Compare, - formatKey: opts.Comparer.FormatKey, - equal: opts.Comparer.Equal, - arenaBuf: opts.arenaBuf, - writerRefs: 1, - logSeqNum: opts.logSeqNum, +func (m *memTable) init(opts memTableOptions) { + if opts.size == 0 { + opts.size = int(opts.MemTableSize) } + *m = memTable{ + cmp: opts.Comparer.Compare, + formatKey: opts.Comparer.FormatKey, + equal: opts.Comparer.Equal, + arenaBuf: opts.arenaBuf, + logSeqNum: opts.logSeqNum, + releaseAccountingReservation: opts.releaseAccountingReservation, + } + m.writerRefs.Store(1) m.tombstones = keySpanCache{ cmp: m.cmp, formatKey: m.formatKey, @@ -144,18 +160,18 @@ func newMemTable(opts memTableOptions) *memTable { m.rangeDelSkl.Reset(arena, m.cmp) m.rangeKeySkl.Reset(arena, m.cmp) m.reserved = arena.Size() - return m } func (m *memTable) writerRef() { - switch v := atomic.AddInt32(&m.writerRefs, 1); { + switch v := m.writerRefs.Add(1); { case v <= 1: panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v)) } } -func (m *memTable) writerUnref() bool { - switch v := atomic.AddInt32(&m.writerRefs, -1); { +// writerUnref drops a ref on the memtable. Returns true if this was the last ref. +func (m *memTable) writerUnref() (wasLastRef bool) { + switch v := m.writerRefs.Add(-1); { case v < 0: panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v)) case v == 0: @@ -165,8 +181,9 @@ func (m *memTable) writerUnref() bool { } } +// readyForFlush is part of the flushable interface. func (m *memTable) readyForFlush() bool { - return atomic.LoadInt32(&m.writerRefs) == 0 + return m.writerRefs.Load() == 0 } // Prepare reserves space for the batch in the memtable and references the @@ -194,11 +211,13 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error { var tombstoneCount, rangeKeyCount uint32 startSeqNum := seqNum for r := batch.Reader(); ; seqNum++ { - kind, ukey, value, ok := r.Next() + kind, ukey, value, ok, err := r.Next() if !ok { + if err != nil { + return err + } break } - var err error ikey := base.MakeInternalKey(ukey, seqNum, kind) switch kind { case InternalKeyKindRangeDelete: @@ -233,17 +252,19 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error { return nil } -// newIter returns an iterator that is unpositioned (Iterator.Valid() will -// return false). The iterator can be positioned via a call to SeekGE, -// SeekLT, First or Last. +// newIter is part of the flushable interface. It returns an iterator that is +// unpositioned (Iterator.Valid() will return false). The iterator can be +// positioned via a call to SeekGE, SeekLT, First or Last. func (m *memTable) newIter(o *IterOptions) internalIterator { return m.skl.NewIter(o.GetLowerBound(), o.GetUpperBound()) } +// newFlushIter is part of the flushable interface. func (m *memTable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator { return m.skl.NewFlushIter(bytesFlushed) } +// newRangeDelIter is part of the flushable interface. func (m *memTable) newRangeDelIter(*IterOptions) keyspan.FragmentIterator { tombstones := m.tombstones.get() if tombstones == nil { @@ -252,6 +273,7 @@ func (m *memTable) newRangeDelIter(*IterOptions) keyspan.FragmentIterator { return keyspan.NewIter(m.cmp, tombstones) } +// newRangeKeyIter is part of the flushable interface. func (m *memTable) newRangeKeyIter(*IterOptions) keyspan.FragmentIterator { rangeKeys := m.rangeKeys.get() if rangeKeys == nil { @@ -260,13 +282,14 @@ func (m *memTable) newRangeKeyIter(*IterOptions) keyspan.FragmentIterator { return keyspan.NewIter(m.cmp, rangeKeys) } +// containsRangeKeys is part of the flushable interface. func (m *memTable) containsRangeKeys() bool { - return atomic.LoadUint32(&m.rangeKeys.atomicCount) > 0 + return m.rangeKeys.count.Load() > 0 } func (m *memTable) availBytes() uint32 { a := m.skl.Arena() - if atomic.LoadInt32(&m.writerRefs) == 1 { + if m.writerRefs.Load() == 1 { // If there are no other concurrent apply operations, we can update the // reserved bytes setting to accurately reflect how many bytes of been // allocated vs the over-estimation present in memTableEntrySize. @@ -275,10 +298,12 @@ func (m *memTable) availBytes() uint32 { return a.Capacity() - m.reserved } +// inuseBytes is part of the flushable interface. func (m *memTable) inuseBytes() uint64 { return uint64(m.skl.Size() - memTableEmptySize) } +// totalBytes is part of the flushable interface. func (m *memTable) totalBytes() uint64 { return uint64(m.skl.Arena().Capacity()) } @@ -355,8 +380,8 @@ func (f *keySpanFrags) get( // invalidated whenever a key of the same kind is added to a memTable, and // populated when empty when a span iterator of that key kind is created. type keySpanCache struct { - atomicCount uint32 - frags unsafe.Pointer + count atomic.Uint32 + frags atomic.Pointer[keySpanFrags] cmp Compare formatKey base.FormatKey constructSpan constructSpan @@ -366,23 +391,20 @@ type keySpanCache struct { // Invalidate the current set of cached spans, indicating the number of // spans that were added. func (c *keySpanCache) invalidate(count uint32) { - newCount := atomic.AddUint32(&c.atomicCount, count) + newCount := c.count.Add(count) var frags *keySpanFrags for { - oldPtr := atomic.LoadPointer(&c.frags) - if oldPtr != nil { - oldFrags := (*keySpanFrags)(oldPtr) - if oldFrags.count >= newCount { - // Someone else invalidated the cache before us and their invalidation - // subsumes ours. - break - } + oldFrags := c.frags.Load() + if oldFrags != nil && oldFrags.count >= newCount { + // Someone else invalidated the cache before us and their invalidation + // subsumes ours. + break } if frags == nil { frags = &keySpanFrags{count: newCount} } - if atomic.CompareAndSwapPointer(&c.frags, oldPtr, unsafe.Pointer(frags)) { + if c.frags.CompareAndSwap(oldFrags, frags) { // We successfully invalidated the cache. break } @@ -391,7 +413,7 @@ func (c *keySpanCache) invalidate(count uint32) { } func (c *keySpanCache) get() []keyspan.Span { - frags := (*keySpanFrags)(atomic.LoadPointer(&c.frags)) + frags := c.frags.Load() if frags == nil { return nil } diff --git a/vendor/github.com/cockroachdb/pebble/merging_iter.go b/vendor/github.com/cockroachdb/pebble/merging_iter.go index 4e2426d..a076686 100644 --- a/vendor/github.com/cockroachdb/pebble/merging_iter.go +++ b/vendor/github.com/cockroachdb/pebble/merging_iter.go @@ -27,6 +27,10 @@ type mergingIterLevel struct { // iterKey and iterValue cache the current key and value iter are pointed at. iterKey *InternalKey iterValue base.LazyValue + // levelIter is non-nil if this level's iter is ultimately backed by a + // *levelIter. The handle in iter may have wrapped the levelIter with + // intermediary internalIterator implementations. + levelIter *levelIter // levelIterBoundaryContext's fields are set when using levelIter, in order // to surface sstable boundary keys and file-level context. See levelIter @@ -1371,9 +1375,9 @@ func (m *mergingIter) DebugString() string { } func (m *mergingIter) ForEachLevelIter(fn func(li *levelIter) bool) { - for _, iter := range m.levels { - if li, ok := iter.iter.(*levelIter); ok { - if done := fn(li); done { + for _, ml := range m.levels { + if ml.levelIter != nil { + if done := fn(ml.levelIter); done { break } } diff --git a/vendor/github.com/cockroachdb/pebble/metrics.go b/vendor/github.com/cockroachdb/pebble/metrics.go index 425b907..cab9291 100644 --- a/vendor/github.com/cockroachdb/pebble/metrics.go +++ b/vendor/github.com/cockroachdb/pebble/metrics.go @@ -6,12 +6,13 @@ package pebble import ( "fmt" - "strings" + "math" "time" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/cache" "github.com/cockroachdb/pebble/internal/humanize" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache" "github.com/cockroachdb/pebble/record" "github.com/cockroachdb/pebble/sstable" "github.com/cockroachdb/redact" @@ -28,13 +29,10 @@ type FilterMetrics = sstable.FilterMetrics // comment in base. type ThroughputMetric = base.ThroughputMetric -func formatCacheMetrics(w redact.SafePrinter, m *CacheMetrics, name redact.SafeString) { - w.Printf("%7s %9s %7s %6.1f%% (score == hit-rate)\n", - name, - humanize.SI.Int64(m.Count), - humanize.IEC.Int64(m.Size), - redact.Safe(hitRate(m.Hits, m.Misses))) -} +// SecondaryCacheMetrics holds metrics for the persistent secondary cache +// that caches commonly accessed blocks from blob storage on a local +// file system. +type SecondaryCacheMetrics = sharedcache.Metrics // LevelMetrics holds per-level metrics such as the number of files and total // size of the files, and compaction related metrics. @@ -46,9 +44,14 @@ type LevelMetrics struct { Sublevels int32 // The total number of files in the level. NumFiles int64 + // The total number of virtual sstables in the level. + NumVirtualFiles uint64 // The total size in bytes of the files in the level. Size int64 - // The level's compaction score. + // The total size of the virtual sstables in the level. + VirtualSize uint64 + // The level's compaction score. This is the compensatedScoreRatio in the + // candidateLevelInfo. Score float64 // The number of incoming bytes from other levels read during // compactions. This excludes bytes moved and bytes ingested. For L0 this is @@ -79,6 +82,18 @@ type LevelMetrics struct { TablesIngested uint64 // The number of sstables moved to this level by a "move" compaction. TablesMoved uint64 + + MultiLevel struct { + // BytesInTop are the total bytes in a multilevel compaction coming from the top level. + BytesInTop uint64 + + // BytesIn, exclusively for multiLevel compactions. + BytesIn uint64 + + // BytesRead, exclusively for multilevel compactions. + BytesRead uint64 + } + // Additional contains misc additional metrics that are not always printed. Additional struct { // The sum of Properties.ValueBlocksSize for all the sstables in this @@ -96,6 +111,8 @@ type LevelMetrics struct { // Add updates the counter metrics for the level. func (m *LevelMetrics) Add(u *LevelMetrics) { m.NumFiles += u.NumFiles + m.NumVirtualFiles += u.NumVirtualFiles + m.VirtualSize += u.VirtualSize m.Size += u.Size m.BytesIn += u.BytesIn m.BytesIngested += u.BytesIngested @@ -107,6 +124,9 @@ func (m *LevelMetrics) Add(u *LevelMetrics) { m.TablesFlushed += u.TablesFlushed m.TablesIngested += u.TablesIngested m.TablesMoved += u.TablesMoved + m.MultiLevel.BytesInTop += u.MultiLevel.BytesInTop + m.MultiLevel.BytesRead += u.MultiLevel.BytesRead + m.MultiLevel.BytesIn += u.MultiLevel.BytesIn m.Additional.BytesWrittenDataBlocks += u.Additional.BytesWrittenDataBlocks m.Additional.BytesWrittenValueBlocks += u.Additional.BytesWrittenValueBlocks m.Additional.ValueBlocksSize += u.Additional.ValueBlocksSize @@ -121,32 +141,6 @@ func (m *LevelMetrics) WriteAmp() float64 { return float64(m.BytesFlushed+m.BytesCompacted) / float64(m.BytesIn) } -// format generates a string of the receiver's metrics, formatting it into the -// supplied buffer. -func (m *LevelMetrics) format( - w redact.SafePrinter, score redact.SafeValue, includeValueBlocksSize bool, -) { - w.Printf("%9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7d %7.1f", - redact.Safe(m.NumFiles), - humanize.IEC.Int64(m.Size), - score, - humanize.IEC.Uint64(m.BytesIn), - humanize.IEC.Uint64(m.BytesIngested), - humanize.SI.Uint64(m.TablesIngested), - humanize.IEC.Uint64(m.BytesMoved), - humanize.SI.Uint64(m.TablesMoved), - humanize.IEC.Uint64(m.BytesFlushed+m.BytesCompacted), - humanize.SI.Uint64(m.TablesFlushed+m.TablesCompacted), - humanize.IEC.Uint64(m.BytesRead), - redact.Safe(m.Sublevels), - redact.Safe(m.WriteAmp())) - if includeValueBlocksSize { - w.Printf(" %7s\n", humanize.IEC.Uint64(m.Additional.ValueBlocksSize)) - } else { - w.SafeString("\n") - } -} - // Metrics holds metrics for various subsystems of the DB such as the Cache, // Compactions, WAL, and per-Level metrics. // @@ -158,14 +152,15 @@ type Metrics struct { Compact struct { // The total number of compactions, and per-compaction type counts. - Count int64 - DefaultCount int64 - DeleteOnlyCount int64 - ElisionOnlyCount int64 - MoveCount int64 - ReadCount int64 - RewriteCount int64 - MultiLevelCount int64 + Count int64 + DefaultCount int64 + DeleteOnlyCount int64 + ElisionOnlyCount int64 + MoveCount int64 + ReadCount int64 + RewriteCount int64 + MultiLevelCount int64 + CounterLevelCount int64 // An estimate of the number of bytes that need to be compacted for the LSM // to reach a stable state. EstimatedDebt uint64 @@ -179,6 +174,14 @@ type Metrics struct { // compaction. Such files are compacted in a rewrite compaction // when no other compactions are picked. MarkedFiles int + // Duration records the cumulative duration of all compactions since the + // database was opened. + Duration time.Duration + } + + Ingest struct { + // The total number of ingestions + Count uint64 } Flush struct { @@ -210,7 +213,10 @@ type Metrics struct { // The count of memtables. Count int64 // The number of bytes present in zombie memtables which are no longer - // referenced by the current DB state but are still in use by an iterator. + // referenced by the current DB state. An unbounded number of memtables + // may be zombie if they're still in use by an iterator. One additional + // memtable may be zombie if it's no longer in use and waiting to be + // recycled. ZombieSize uint64 // The count of zombie memtables. ZombieCount int64 @@ -222,6 +228,9 @@ type Metrics struct { // The approximate count of internal tombstones (DEL, SINGLEDEL and // RANGEDEL key kinds) within the database. TombstoneCount uint64 + // A cumulative total number of missized DELSIZED keys encountered by + // compactions since the database was opened. + MissizedTombstonesCount uint64 } Snapshots struct { @@ -250,12 +259,18 @@ type Metrics struct { ZombieSize uint64 // The count of zombie tables. ZombieCount int64 + // The count of the backing sstables. + BackingTableCount uint64 + // The sum of the sizes of the all of the backing sstables. + BackingTableSize uint64 } TableCache CacheMetrics // Count of the number of open sstable iterators. TableIters int64 + // Uptime is the total time since this DB was opened. + Uptime time.Duration WAL struct { // Number of live WAL files. @@ -281,6 +296,8 @@ type Metrics struct { record.LogWriterMetrics } + SecondaryCacheMetrics SecondaryCacheMetrics + private struct { optionsFileSize uint64 manifestFileSize uint64 @@ -294,6 +311,13 @@ var ( prometheus.LinearBuckets(0.0, float64(time.Microsecond*100), 50), prometheus.ExponentialBucketsRange(float64(time.Millisecond*5), float64(10*time.Second), 50)..., ) + + // SecondaryCacheIOBuckets exported to enable exporting from package pebble to + // enable exporting metrics with below buckets in CRDB. + SecondaryCacheIOBuckets = sharedcache.IOBuckets + // SecondaryCacheChannelWriteBuckets exported to enable exporting from package + // pebble to enable exporting metrics with below buckets in CRDB. + SecondaryCacheChannelWriteBuckets = sharedcache.ChannelWriteBuckets ) // DiskSpaceUsage returns the total disk space used by the database in bytes, @@ -313,12 +337,25 @@ func (m *Metrics) DiskSpaceUsage() uint64 { return usageBytes } -func (m *Metrics) levelSizes() [numLevels]int64 { - var sizes [numLevels]int64 - for i := 0; i < len(sizes); i++ { - sizes[i] = m.Levels[i].Size +// NumVirtual is the number of virtual sstables in the latest version +// summed over every level in the lsm. +func (m *Metrics) NumVirtual() uint64 { + var n uint64 + for _, level := range m.Levels { + n += level.NumVirtualFiles } - return sizes + return n +} + +// VirtualSize is the sum of the sizes of the virtual sstables in the +// latest version. BackingTableSize - VirtualSize gives an estimate for +// the space amplification caused by not compacting virtual sstables. +func (m *Metrics) VirtualSize() uint64 { + var size uint64 + for _, level := range m.Levels { + size += level.VirtualSize + } + return size } // ReadAmp returns the current read amplification of the database. @@ -349,59 +386,34 @@ func (m *Metrics) Total() LevelMetrics { return total } -const notApplicable = redact.SafeString("-") - -func (m *Metrics) formatWAL(w redact.SafePrinter) { - var writeAmp float64 - if m.WAL.BytesIn > 0 { - writeAmp = float64(m.WAL.BytesWritten) / float64(m.WAL.BytesIn) - } - w.Printf(" WAL %9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7.1f\n", - redact.Safe(m.WAL.Files), - humanize.Uint64(m.WAL.Size), - notApplicable, - humanize.Uint64(m.WAL.BytesIn), - notApplicable, - notApplicable, - notApplicable, - notApplicable, - humanize.Uint64(m.WAL.BytesWritten), - notApplicable, - notApplicable, - notApplicable, - redact.Safe(writeAmp)) -} - -// String pretty-prints the metrics, showing a line for the WAL, a line per-level, and -// a total: +// String pretty-prints the metrics as below: // -// __level_____count____size___score______in__ingest(sz_cnt)____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp -// WAL 1 28 B - 17 B - - - - 56 B - - - 3.3 -// 0 1 770 B 0.25 28 B 0 B 0 0 B 0 770 B 1 0 B 1 27.5 -// 1 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0 -// 2 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0 -// 3 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0 -// 4 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0 -// 5 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0 -// 6 0 0 B - 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0 -// total 1 770 B - 56 B 0 B 0 0 B 0 826 B 1 0 B 1 14.8 -// flush 1 0 B 0 0 (ingest = ingested-as-flushable, move = tables-ingested) -// compact 0 0 B 0 B 0 (size == estimated-debt, score = in-progress-bytes, in = num-in-progress) -// ctype 0 0 0 0 0 0 0 (default, delete, elision, move, read, rewrite, multi-level) -// memtbl 1 256 K -// zmemtbl 1 256 K -// ztbl 0 0 B -// bcache 4 697 B 0.0% (score == hit-rate) -// tcache 1 696 B 0.0% (score == hit-rate) -// snaps 0 - 0 (score == earliest seq num) -// titers 1 -// filter - - 0.0% (score == utility) -// -// The WAL "in" metric is the size of the batches written to the WAL. The WAL -// "write" metric is the size of the physical data written to the WAL which -// includes record fragment overhead. Write amplification is computed as -// bytes-written / bytes-in, except for the total row where bytes-in is -// replaced with WAL-bytes-written + bytes-ingested. +// | | | | ingested | moved | written | | amp +// level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w +// ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- +// 0 | 101 102B 0B 0 | 103.0 | 104B | 112 104B | 113 106B | 221 217B | 107B | 1 2.1 +// 1 | 201 202B 0B 0 | 203.0 | 204B | 212 204B | 213 206B | 421 417B | 207B | 2 2.0 +// 2 | 301 302B 0B 0 | 303.0 | 304B | 312 304B | 313 306B | 621 617B | 307B | 3 2.0 +// 3 | 401 402B 0B 0 | 403.0 | 404B | 412 404B | 413 406B | 821 817B | 407B | 4 2.0 +// 4 | 501 502B 0B 0 | 503.0 | 504B | 512 504B | 513 506B | 1.0K 1017B | 507B | 5 2.0 +// 5 | 601 602B 0B 0 | 603.0 | 604B | 612 604B | 613 606B | 1.2K 1.2KB | 607B | 6 2.0 +// 6 | 701 702B 0B 0 | - | 704B | 712 704B | 713 706B | 1.4K 1.4KB | 707B | 7 2.0 +// total | 2.8K 2.7KB 0B 0 | - | 2.8KB | 2.9K 2.8KB | 2.9K 2.8KB | 5.7K 8.4KB | 2.8KB | 28 3.0 +// ------------------------------------------------------------------------------------------------------------------- +// WAL: 22 files (24B) in: 25B written: 26B (4% overhead) +// Flushes: 8 +// Compactions: 5 estimated debt: 6B in progress: 2 (7B) +// default: 27 delete: 28 elision: 29 move: 30 read: 31 rewrite: 32 multi-level: 33 +// MemTables: 12 (11B) zombie: 14 (13B) +// Zombie tables: 16 (15B) +// Backing tables: 0 (0B) +// Block cache: 2 entries (1B) hit rate: 42.9% +// Table cache: 18 entries (17B) hit rate: 48.7% +// Secondary cache: 40 entries (40B) hit rate: 49.9% +// Snapshots: 4 earliest seq num: 1024 +// Table iters: 21 +// Filter utility: 47.4% +// Ingestions: 27 as flushable: 36 (34B in 35 tables) func (m *Metrics) String() string { return redact.StringWithoutMarkers(m) } @@ -422,29 +434,84 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) { // width specifiers. When the issue is fixed, we can convert these to // RedactableStrings. https://github.com/cockroachdb/redact/issues/17 - haveValueBlocks := false - var valueBlocksHeading redact.SafeString - for level := 0; level < numLevels; level++ { - if m.Levels[level].Additional.ValueBlocksSize > 0 { - haveValueBlocks = true - valueBlocksHeading = "__val-bl" - break + multiExists := m.Compact.MultiLevelCount > 0 + appendIfMulti := func(line redact.SafeString) { + if multiExists { + w.SafeString(line) } } + newline := func() { + w.SafeString("\n") + } + + w.SafeString(" | | | | ingested | moved | written | | amp") + appendIfMulti(" | multilevel") + newline() + w.SafeString("level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w") + appendIfMulti(" | top in read") + newline() + w.SafeString("------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+---------") + appendIfMulti("-+------------------") + newline() + + // formatRow prints out a row of the table. + formatRow := func(m *LevelMetrics, score float64) { + scoreStr := "-" + if !math.IsNaN(score) { + // Try to keep the string no longer than 5 characters. + switch { + case score < 99.995: + scoreStr = fmt.Sprintf("%.2f", score) + case score < 999.95: + scoreStr = fmt.Sprintf("%.1f", score) + default: + scoreStr = fmt.Sprintf("%.0f", score) + } + } + var wampStr string + if wamp := m.WriteAmp(); wamp > 99.5 { + wampStr = fmt.Sprintf("%.0f", wamp) + } else { + wampStr = fmt.Sprintf("%.1f", wamp) + } + + w.Printf("| %5s %6s %6s %7s | %5s | %5s | %5s %6s | %5s %6s | %5s %6s | %5s | %3d %4s", + humanize.Count.Int64(m.NumFiles), + humanize.Bytes.Int64(m.Size), + humanize.Bytes.Uint64(m.Additional.ValueBlocksSize), + humanize.Count.Uint64(m.NumVirtualFiles), + redact.Safe(scoreStr), + humanize.Bytes.Uint64(m.BytesIn), + humanize.Count.Uint64(m.TablesIngested), + humanize.Bytes.Uint64(m.BytesIngested), + humanize.Count.Uint64(m.TablesMoved), + humanize.Bytes.Uint64(m.BytesMoved), + humanize.Count.Uint64(m.TablesFlushed+m.TablesCompacted), + humanize.Bytes.Uint64(m.BytesFlushed+m.BytesCompacted), + humanize.Bytes.Uint64(m.BytesRead), + redact.Safe(m.Sublevels), + redact.Safe(wampStr)) + + if multiExists { + w.Printf(" | %5s %5s %5s", + humanize.Bytes.Uint64(m.MultiLevel.BytesInTop), + humanize.Bytes.Uint64(m.MultiLevel.BytesIn), + humanize.Bytes.Uint64(m.MultiLevel.BytesRead)) + } + newline() + } + var total LevelMetrics - w.Printf("__level_____count____size___score______in__ingest(sz_cnt)"+ - "____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp%s\n", valueBlocksHeading) - m.formatWAL(w) for level := 0; level < numLevels; level++ { l := &m.Levels[level] - w.Printf("%7d ", redact.Safe(level)) + w.Printf("%5d ", redact.Safe(level)) // Format the score. - var score redact.SafeValue = notApplicable + score := math.NaN() if level < numLevels-1 { - score = redact.Safe(fmt.Sprintf("%0.2f", l.Score)) + score = l.Score } - l.format(w, score, haveValueBlocks) + formatRow(l, score) total.Add(l) total.Sublevels += l.Sublevels } @@ -454,57 +521,105 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) { // the bytes written to the log and bytes written externally and then // ingested. total.BytesFlushed += total.BytesIn - w.SafeString(" total ") - total.format(w, notApplicable, haveValueBlocks) + w.SafeString("total ") + formatRow(&total, math.NaN()) - w.Printf(" flush %9d %31s %7d %7d %s\n", - redact.Safe(m.Flush.Count), - humanize.IEC.Uint64(m.Flush.AsIngestBytes), - redact.Safe(m.Flush.AsIngestTableCount), - redact.Safe(m.Flush.AsIngestCount), - redact.SafeString(`(ingest = tables-ingested, move = ingested-as-flushable)`)) - w.Printf("compact %9d %7s %7s %7d %s %s\n", + w.SafeString("-------------------------------------------------------------------------------------------------------------------") + appendIfMulti("--------------------") + newline() + w.Printf("WAL: %d files (%s) in: %s written: %s (%.0f%% overhead)\n", + redact.Safe(m.WAL.Files), + humanize.Bytes.Uint64(m.WAL.Size), + humanize.Bytes.Uint64(m.WAL.BytesIn), + humanize.Bytes.Uint64(m.WAL.BytesWritten), + redact.Safe(percent(int64(m.WAL.BytesWritten)-int64(m.WAL.BytesIn), int64(m.WAL.BytesIn)))) + + w.Printf("Flushes: %d\n", redact.Safe(m.Flush.Count)) + + w.Printf("Compactions: %d estimated debt: %s in progress: %d (%s)\n", redact.Safe(m.Compact.Count), - humanize.IEC.Uint64(m.Compact.EstimatedDebt), - humanize.IEC.Int64(m.Compact.InProgressBytes), + humanize.Bytes.Uint64(m.Compact.EstimatedDebt), redact.Safe(m.Compact.NumInProgress), - redact.SafeString(strings.Repeat(" ", 24)), - redact.SafeString(`(size == estimated-debt, score = in-progress-bytes, in = num-in-progress)`)) - w.Printf(" ctype %9d %7d %7d %7d %7d %7d %7d %s\n", + humanize.Bytes.Int64(m.Compact.InProgressBytes)) + + w.Printf(" default: %d delete: %d elision: %d move: %d read: %d rewrite: %d multi-level: %d\n", redact.Safe(m.Compact.DefaultCount), redact.Safe(m.Compact.DeleteOnlyCount), redact.Safe(m.Compact.ElisionOnlyCount), redact.Safe(m.Compact.MoveCount), redact.Safe(m.Compact.ReadCount), redact.Safe(m.Compact.RewriteCount), - redact.Safe(m.Compact.MultiLevelCount), - redact.SafeString(`(default, delete, elision, move, read, rewrite, multi-level)`)) - w.Printf(" memtbl %9d %7s\n", + redact.Safe(m.Compact.MultiLevelCount)) + + w.Printf("MemTables: %d (%s) zombie: %d (%s)\n", redact.Safe(m.MemTable.Count), - humanize.IEC.Uint64(m.MemTable.Size)) - w.Printf("zmemtbl %9d %7s\n", + humanize.Bytes.Uint64(m.MemTable.Size), redact.Safe(m.MemTable.ZombieCount), - humanize.IEC.Uint64(m.MemTable.ZombieSize)) - w.Printf(" ztbl %9d %7s\n", + humanize.Bytes.Uint64(m.MemTable.ZombieSize)) + + w.Printf("Zombie tables: %d (%s)\n", redact.Safe(m.Table.ZombieCount), - humanize.IEC.Uint64(m.Table.ZombieSize)) - formatCacheMetrics(w, &m.BlockCache, "bcache") - formatCacheMetrics(w, &m.TableCache, "tcache") - w.Printf(" snaps %9d %7s %7d (score == earliest seq num)\n", + humanize.Bytes.Uint64(m.Table.ZombieSize)) + + w.Printf("Backing tables: %d (%s)\n", + redact.Safe(m.Table.BackingTableCount), + humanize.Bytes.Uint64(m.Table.BackingTableSize)) + w.Printf("Virtual tables: %d (%s)\n", + redact.Safe(m.NumVirtual()), + humanize.Bytes.Uint64(m.VirtualSize())) + + formatCacheMetrics := func(m *CacheMetrics, name redact.SafeString) { + w.Printf("%s: %s entries (%s) hit rate: %.1f%%\n", + name, + humanize.Count.Int64(m.Count), + humanize.Bytes.Int64(m.Size), + redact.Safe(hitRate(m.Hits, m.Misses))) + } + formatCacheMetrics(&m.BlockCache, "Block cache") + formatCacheMetrics(&m.TableCache, "Table cache") + + formatSharedCacheMetrics := func(w redact.SafePrinter, m *SecondaryCacheMetrics, name redact.SafeString) { + w.Printf("%s: %s entries (%s) hit rate: %.1f%%\n", + name, + humanize.Count.Int64(m.Count), + humanize.Bytes.Int64(m.Size), + redact.Safe(hitRate(m.ReadsWithFullHit, m.ReadsWithPartialHit+m.ReadsWithNoHit))) + } + formatSharedCacheMetrics(w, &m.SecondaryCacheMetrics, "Secondary cache") + + w.Printf("Snapshots: %d earliest seq num: %d\n", redact.Safe(m.Snapshots.Count), - notApplicable, redact.Safe(m.Snapshots.EarliestSeqNum)) - w.Printf(" titers %9d\n", redact.Safe(m.TableIters)) - w.Printf(" filter %9s %7s %6.1f%% (score == utility)\n", - notApplicable, - notApplicable, - redact.Safe(hitRate(m.Filter.Hits, m.Filter.Misses))) + + w.Printf("Table iters: %d\n", redact.Safe(m.TableIters)) + w.Printf("Filter utility: %.1f%%\n", redact.Safe(hitRate(m.Filter.Hits, m.Filter.Misses))) + w.Printf("Ingestions: %d as flushable: %d (%s in %d tables)\n", + redact.Safe(m.Ingest.Count), + redact.Safe(m.Flush.AsIngestCount), + humanize.Bytes.Uint64(m.Flush.AsIngestBytes), + redact.Safe(m.Flush.AsIngestTableCount)) } func hitRate(hits, misses int64) float64 { - sum := hits + misses - if sum == 0 { + return percent(hits, hits+misses) +} + +func percent(numerator, denominator int64) float64 { + if denominator == 0 { return 0 } - return 100 * float64(hits) / float64(sum) + return 100 * float64(numerator) / float64(denominator) +} + +// StringForTests is identical to m.String() on 64-bit platforms. It is used to +// provide a platform-independent result for tests. +func (m *Metrics) StringForTests() string { + mCopy := *m + if math.MaxInt == math.MaxInt32 { + // This is the difference in Sizeof(sstable.Reader{})) between 64 and 32 bit + // platforms. + const tableCacheSizeAdjustment = 212 + mCopy.TableCache.Size += mCopy.TableCache.Count * tableCacheSizeAdjustment + } + return redact.StringWithoutMarkers(&mCopy) } diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/noop_readahead.go b/vendor/github.com/cockroachdb/pebble/objstorage/noop_readahead.go index beb0d92..72c0a85 100644 --- a/vendor/github.com/cockroachdb/pebble/objstorage/noop_readahead.go +++ b/vendor/github.com/cockroachdb/pebble/objstorage/noop_readahead.go @@ -4,26 +4,31 @@ package objstorage -import "io" +import "context" // NoopReadHandle can be used by Readable implementations that don't // support read-ahead. type NoopReadHandle struct { - io.ReaderAt + readable Readable } // MakeNoopReadHandle initializes a NoopReadHandle. -func MakeNoopReadHandle(r io.ReaderAt) NoopReadHandle { - return NoopReadHandle{ReaderAt: r} +func MakeNoopReadHandle(r Readable) NoopReadHandle { + return NoopReadHandle{readable: r} } var _ ReadHandle = (*NoopReadHandle)(nil) +// ReadAt is part of the ReadHandle interface. +func (h *NoopReadHandle) ReadAt(ctx context.Context, p []byte, off int64) error { + return h.readable.ReadAt(ctx, p, off) +} + // Close is part of the ReadHandle interface. func (*NoopReadHandle) Close() error { return nil } -// MaxReadahead is part of the ReadHandle interface. -func (*NoopReadHandle) MaxReadahead() {} +// SetupForCompaction is part of the ReadHandle interface. +func (*NoopReadHandle) SetupForCompaction() {} // RecordCacheHit is part of the ReadHandle interface. -func (*NoopReadHandle) RecordCacheHit(offset, size int64) {} +func (*NoopReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorage.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorage.go new file mode 100644 index 0000000..bbb5b6e --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorage.go @@ -0,0 +1,324 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorage + +import ( + "context" + "fmt" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache" + "github.com/cockroachdb/pebble/objstorage/remote" + "github.com/cockroachdb/pebble/vfs" +) + +// Readable is the handle for an object that is open for reading. +type Readable interface { + // ReadAt reads len(p) bytes into p starting at offset off. + // + // Does not return partial results; if off + len(p) is past the end of the + // object, an error is returned. + // + // Clients of ReadAt can execute parallel ReadAt calls on the + // same Readable. + ReadAt(ctx context.Context, p []byte, off int64) error + + Close() error + + // Size returns the size of the object. + Size() int64 + + // NewReadHandle creates a read handle for ReadAt requests that are related + // and can benefit from optimizations like read-ahead. + // + // The ReadHandle must be closed before the Readable is closed. + // + // Multiple separate ReadHandles can be used. + NewReadHandle(ctx context.Context) ReadHandle +} + +// ReadHandle is used to perform reads that are related and might benefit from +// optimizations like read-ahead. +type ReadHandle interface { + // ReadAt reads len(p) bytes into p starting at offset off. + // + // Does not return partial results; if off + len(p) is past the end of the + // object, an error is returned. + // + // Parallel ReadAt calls on the same ReadHandle are not allowed. + ReadAt(ctx context.Context, p []byte, off int64) error + + Close() error + + // SetupForCompaction informs the implementation that the read handle will + // be used to read data blocks for a compaction. The implementation can expect + // sequential reads, and can decide to not retain data in any caches. + SetupForCompaction() + + // RecordCacheHit informs the implementation that we were able to retrieve a + // block from cache. This is useful for example when the implementation is + // trying to detect a sequential reading pattern. + RecordCacheHit(ctx context.Context, offset, size int64) +} + +// Writable is the handle for an object that is open for writing. +// Either Finish or Abort must be called. +type Writable interface { + // Write writes len(p) bytes from p to the underlying object. The data is not + // guaranteed to be durable until Finish is called. + // + // Note that Write *is* allowed to modify the slice passed in, whether + // temporarily or permanently. Callers of Write need to take this into + // account. + Write(p []byte) error + + // Finish completes the object and makes the data durable. + // No further calls are allowed after calling Finish. + Finish() error + + // Abort gives up on finishing the object. There is no guarantee about whether + // the object exists after calling Abort. + // No further calls are allowed after calling Abort. + Abort() +} + +// ObjectMetadata contains the metadata required to be able to access an object. +type ObjectMetadata struct { + DiskFileNum base.DiskFileNum + FileType base.FileType + + // The fields below are only set if the object is on remote storage. + Remote struct { + // CreatorID identifies the DB instance that originally created the object. + // + // Only used when CustomObjectName is not set. + CreatorID CreatorID + // CreatorFileNum is the identifier for the object within the context of the + // DB instance that originally created the object. + // + // Only used when CustomObjectName is not set. + CreatorFileNum base.DiskFileNum + // CustomObjectName (if it is set) overrides the object name that is normally + // derived from the CreatorID and CreatorFileNum. + CustomObjectName string + // CleanupMethod indicates the method for cleaning up unused shared objects. + CleanupMethod SharedCleanupMethod + // Locator identifies the remote.Storage implementation for this object. + Locator remote.Locator + // Storage is the remote.Storage object corresponding to the Locator. Used + // to avoid lookups in hot paths. + Storage remote.Storage + } +} + +// IsRemote returns true if the object is on remote storage. +func (meta *ObjectMetadata) IsRemote() bool { + return meta.IsShared() || meta.IsExternal() +} + +// IsExternal returns true if the object is on remote storage but is not owned +// by any Pebble instances in the cluster. +func (meta *ObjectMetadata) IsExternal() bool { + return meta.Remote.CustomObjectName != "" +} + +// IsShared returns true if the object is on remote storage and is owned by a +// Pebble instance in the cluster (potentially shared between multiple +// instances). +func (meta *ObjectMetadata) IsShared() bool { + return meta.Remote.CreatorID.IsSet() +} + +// AssertValid checks that the metadata is sane. +func (meta *ObjectMetadata) AssertValid() { + if !meta.IsRemote() { + // Verify all Remote fields are empty. + if meta.Remote != (ObjectMetadata{}).Remote { + panic(errors.AssertionFailedf("meta.Remote not empty: %#v", meta.Remote)) + } + } else { + if meta.Remote.CustomObjectName != "" { + if meta.Remote.CreatorID == 0 { + panic(errors.AssertionFailedf("CreatorID not set")) + } + if meta.Remote.CreatorFileNum == base.FileNum(0).DiskFileNum() { + panic(errors.AssertionFailedf("CreatorFileNum not set")) + } + } + if meta.Remote.CleanupMethod != SharedNoCleanup && meta.Remote.CleanupMethod != SharedRefTracking { + panic(errors.AssertionFailedf("invalid CleanupMethod %d", meta.Remote.CleanupMethod)) + } + if meta.Remote.Storage == nil { + panic(errors.AssertionFailedf("Storage not set")) + } + } +} + +// CreatorID identifies the DB instance that originally created a shared object. +// This ID is incorporated in backing object names. +// Must be non-zero. +type CreatorID uint64 + +// IsSet returns true if the CreatorID is not zero. +func (c CreatorID) IsSet() bool { return c != 0 } + +func (c CreatorID) String() string { return fmt.Sprintf("%d", c) } + +// SharedCleanupMethod indicates the method for cleaning up unused shared objects. +type SharedCleanupMethod uint8 + +const ( + // SharedRefTracking is used for shared objects for which objstorage providers + // keep track of references via reference marker objects. + SharedRefTracking SharedCleanupMethod = iota + + // SharedNoCleanup is used for remote objects that are managed externally; the + // objstorage provider never deletes such objects. + SharedNoCleanup +) + +// OpenOptions contains optional arguments for OpenForReading. +type OpenOptions struct { + // MustExist triggers a fatal error if the file does not exist. The fatal + // error message contains extra information helpful for debugging. + MustExist bool +} + +// CreateOptions contains optional arguments for Create. +type CreateOptions struct { + // PreferSharedStorage causes the object to be created on shared storage if + // the provider has shared storage configured. + PreferSharedStorage bool + + // SharedCleanupMethod is used for the object when it is created on shared storage. + // The default (zero) value is SharedRefTracking. + SharedCleanupMethod SharedCleanupMethod +} + +// Provider is a singleton object used to access and manage objects. +// +// An object is conceptually like a large immutable file. The main use of +// objects is for storing sstables; in the future it could also be used for blob +// storage. +// +// The Provider can only manage objects that it knows about - either objects +// created by the provider, or existing objects the Provider was informed about +// via AddObjects. +// +// Objects are currently backed by a vfs.File or a remote.Storage object. +type Provider interface { + // OpenForReading opens an existing object. + OpenForReading( + ctx context.Context, fileType base.FileType, FileNum base.DiskFileNum, opts OpenOptions, + ) (Readable, error) + + // Create creates a new object and opens it for writing. + // + // The object is not guaranteed to be durable (accessible in case of crashes) + // until Sync is called. + Create( + ctx context.Context, fileType base.FileType, FileNum base.DiskFileNum, opts CreateOptions, + ) (w Writable, meta ObjectMetadata, err error) + + // Remove removes an object. + // + // The object is not guaranteed to be durably removed until Sync is called. + Remove(fileType base.FileType, FileNum base.DiskFileNum) error + + // Sync flushes the metadata from creation or removal of objects since the last Sync. + // This includes objects that have been Created but for which + // Writable.Finish() has not yet been called. + Sync() error + + // LinkOrCopyFromLocal creates a new object that is either a copy of a given + // local file or a hard link (if the new object is created on the same FS, and + // if the FS supports it). + // + // The object is not guaranteed to be durable (accessible in case of crashes) + // until Sync is called. + LinkOrCopyFromLocal( + ctx context.Context, + srcFS vfs.FS, + srcFilePath string, + dstFileType base.FileType, + dstFileNum base.DiskFileNum, + opts CreateOptions, + ) (ObjectMetadata, error) + + // Lookup returns the metadata of an object that is already known to the Provider. + // Does not perform any I/O. + Lookup(fileType base.FileType, FileNum base.DiskFileNum) (ObjectMetadata, error) + + // Path returns an internal, implementation-dependent path for the object. It is + // meant to be used for informational purposes (like logging). + Path(meta ObjectMetadata) string + + // Size returns the size of the object. + Size(meta ObjectMetadata) (int64, error) + + // List returns the objects currently known to the provider. Does not perform any I/O. + List() []ObjectMetadata + + // SetCreatorID sets the CreatorID which is needed in order to use shared + // objects. Remote object usage is disabled until this method is called the + // first time. Once set, the Creator ID is persisted and cannot change. + // + // Cannot be called if shared storage is not configured for the provider. + SetCreatorID(creatorID CreatorID) error + + // IsSharedForeign returns whether this object is owned by a different node. + IsSharedForeign(meta ObjectMetadata) bool + + // RemoteObjectBacking encodes the remote object metadata for the given object. + RemoteObjectBacking(meta *ObjectMetadata) (RemoteObjectBackingHandle, error) + + // CreateExternalObjectBacking creates a backing for an existing object with a + // custom object name. The object is considered to be managed outside of + // Pebble and will never be removed by Pebble. + CreateExternalObjectBacking(locator remote.Locator, objName string) (RemoteObjectBacking, error) + + // AttachRemoteObjects registers existing remote objects with this provider. + AttachRemoteObjects(objs []RemoteObjectToAttach) ([]ObjectMetadata, error) + + Close() error + + // IsNotExistError indicates whether the error is known to report that a file or + // directory does not exist. + IsNotExistError(err error) bool + + // Metrics returns metrics about objstorage. Currently, it only returns metrics + // about the shared cache. + Metrics() sharedcache.Metrics +} + +// RemoteObjectBacking encodes the metadata necessary to incorporate a shared +// object into a different Pebble instance. The encoding is specific to a given +// Provider implementation. +type RemoteObjectBacking []byte + +// RemoteObjectBackingHandle is a container for a RemoteObjectBacking which +// ensures that the backing stays valid. A backing can otherwise become invalid +// if this provider unrefs the shared object. The RemoteObjectBackingHandle +// delays any unref until Close. +type RemoteObjectBackingHandle interface { + // Get returns the backing. The backing is only guaranteed to be valid until + // Close is called (or until the Provider is closed). If Close was already + // called, returns an error. + Get() (RemoteObjectBacking, error) + Close() +} + +// RemoteObjectToAttach contains the arguments needed to attach an existing remote object. +type RemoteObjectToAttach struct { + // FileNum is the file number that will be used to refer to this object (in + // the context of this instance). + FileNum base.DiskFileNum + FileType base.FileType + // Backing contains the metadata for the remote object backing (normally + // generated from a different instance, but using the same Provider + // implementation). + Backing RemoteObjectBacking +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing.go new file mode 100644 index 0000000..2672636 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing.go @@ -0,0 +1,68 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objiotracing + +import "github.com/cockroachdb/pebble/internal/base" + +// OpType indicates the type of operation. +type OpType uint8 + +// OpType values. +const ( + ReadOp OpType = iota + WriteOp + // RecordCacheHitOp happens when a read is satisfied from the block cache. See + // objstorage.ReadHandle.RecordCacheHit(). + RecordCacheHitOp + // SetupForCompactionOp is a "meta operation" that configures a read handle + // for large sequential reads. See objstorage.ReadHandle.SetupForCompaction(). + SetupForCompactionOp +) + +// Reason indicates the higher-level context of the operation. +type Reason uint8 + +// Reason values. +const ( + UnknownReason Reason = iota + ForFlush + ForCompaction + ForIngestion + // TODO(radu): add ForUserFacing. +) + +// BlockType indicates the type of data block relevant to an operation. +type BlockType uint8 + +// BlockType values. +const ( + UnknownBlock BlockType = iota + DataBlock + ValueBlock + FilterBlock + MetadataBlock +) + +// Event is the on-disk format of a tracing event. It is exported here so that +// trace processing tools can use it by importing this package. +type Event struct { + // Event start time as a Unix time (see time.Time.StartUnixNano()). + // Note that recorded events are not necessarily ordered by time - this is + // because separate event "streams" use local buffers (for performance). + StartUnixNano int64 + Op OpType + Reason Reason + BlockType BlockType + // LSM level plus one (with 0 indicating unknown level). + LevelPlusOne uint8 + // Hardcoded padding so that struct layout doesn't depend on architecture. + _ uint32 + FileNum base.FileNum + // HandleID is a unique identifier corresponding to an objstorage.ReadHandle; + // only set for read operations performed through a ReadHandle. + HandleID uint64 + Offset int64 + Size int64 +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go new file mode 100644 index 0000000..a4923ab --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go @@ -0,0 +1,59 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +//go:build !pebble_obj_io_tracing +// +build !pebble_obj_io_tracing + +package objiotracing + +import ( + "context" + + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/vfs" +) + +// Enabled is used to short circuit tracing-related code in regular builds. +const Enabled = false + +// Tracer manages the writing of object IO traces to files. +type Tracer struct{} + +// Open creates a Tracer which generates trace files in the given directory. +// Each trace file contains a series of Events (as they are in memory). +func Open(fs vfs.FS, fsDir string) *Tracer { + return nil +} + +// Close the tracer, flushing any remaining events. +func (*Tracer) Close() {} + +// WrapReadable wraps an objstorage.Readable with one that generates tracing +// events. +func (*Tracer) WrapReadable( + ctx context.Context, r objstorage.Readable, fileNum base.DiskFileNum, +) objstorage.Readable { + return r +} + +// WrapWritable wraps an objstorage.Writable with one that generates tracing +// events. +func (t *Tracer) WrapWritable( + ctx context.Context, w objstorage.Writable, fileNum base.DiskFileNum, +) objstorage.Writable { + return w +} + +// WithReason creates a context that has an associated Reason (which ends up in +// traces created under that context). +func WithReason(ctx context.Context, reason Reason) context.Context { return ctx } + +// WithBlockType creates a context that has an associated BlockType (which ends up in +// traces created under that context). +func WithBlockType(ctx context.Context, blockType BlockType) context.Context { return ctx } + +// WithLevel creates a context that has an associated level (which ends up in +// traces created under that context). +func WithLevel(ctx context.Context, level int) context.Context { return ctx } diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go new file mode 100644 index 0000000..0680b34 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go @@ -0,0 +1,410 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +//go:build pebble_obj_io_tracing +// +build pebble_obj_io_tracing + +package objiotracing + +import ( + "bufio" + "context" + "fmt" + "math/rand" + "sync" + "sync/atomic" + "time" + "unsafe" + + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/vfs" +) + +// Enabled is used to short circuit tracing-related code in regular builds. +const Enabled = true + +// Tracer manages the writing of object IO traces to files. +// +// The tracer runs a background worker goroutine which receives trace event +// buffers over a channel and dumps them to IOTRACES- files. Wrapper +// implementations for Readable, ReadHandle, Writable are producers of traces; +// they maintain internal buffers of events which get flushed to the buffered +// channel when they get full. This allows for minimal synchronization per IO +// (as for most of these structures, an instance only allows a single IO at a +// time). +type Tracer struct { + fs vfs.FS + fsDir string + + handleID atomic.Uint64 + + workerStopCh chan struct{} + workerDataCh chan eventBuf + workerWait sync.WaitGroup +} + +// Open creates a Tracer which generates trace files in the given directory. +// Each trace file contains a series of Events (as they are in memory). +func Open(fs vfs.FS, fsDir string) *Tracer { + t := &Tracer{ + fs: fs, + fsDir: fsDir, + workerStopCh: make(chan struct{}), + workerDataCh: make(chan eventBuf, channelBufSize), + } + + t.handleID.Store(uint64(rand.NewSource(time.Now().UnixNano()).Int63())) + + t.workerWait.Add(1) + go t.workerLoop() + return t +} + +// Close the tracer, flushing any remaining events. +func (t *Tracer) Close() { + if t.workerStopCh == nil { + return + } + // Tell the worker to stop and wait for it to finish up. + close(t.workerStopCh) + t.workerWait.Wait() + t.workerStopCh = nil +} + +// WrapWritable wraps an objstorage.Writable with one that generates tracing +// events. +func (t *Tracer) WrapWritable( + ctx context.Context, w objstorage.Writable, fileNum base.FileNum, +) objstorage.Writable { + return &writable{ + w: w, + fileNum: fileNum, + g: makeEventGenerator(ctx, t), + } +} + +type writable struct { + w objstorage.Writable + fileNum base.FileNum + curOffset int64 + g eventGenerator +} + +var _ objstorage.Writable = (*writable)(nil) + +// Write is part of the objstorage.Writable interface. +func (w *writable) Write(p []byte) error { + w.g.add(context.Background(), Event{ + Op: WriteOp, + FileNum: w.fileNum, + Offset: w.curOffset, + Size: int64(len(p)), + }) + // If w.w.Write(p) returns an error, a new writable + // will be used, so even tho all of p may not have + // been written to the underlying "file", it is okay + // to add len(p) to curOffset. + w.curOffset += int64(len(p)) + return w.w.Write(p) +} + +// Finish is part of the objstorage.Writable interface. +func (w *writable) Finish() error { + w.g.flush() + return w.w.Finish() +} + +// Abort is part of the objstorage.Writable interface. +func (w *writable) Abort() { + w.g.flush() + w.w.Abort() +} + +// WrapReadable wraps an objstorage.Readable with one that generates tracing +// events. +func (t *Tracer) WrapReadable( + ctx context.Context, r objstorage.Readable, fileNum base.FileNum, +) objstorage.Readable { + res := &readable{ + r: r, + fileNum: fileNum, + } + res.mu.g = makeEventGenerator(ctx, t) + return res +} + +type readable struct { + r objstorage.Readable + fileNum base.FileNum + mu struct { + sync.Mutex + g eventGenerator + } +} + +var _ objstorage.Readable = (*readable)(nil) + +// ReadAt is part of the objstorage.Readable interface. +func (r *readable) ReadAt(ctx context.Context, v []byte, off int64) (n int, err error) { + r.mu.Lock() + r.mu.g.add(ctx, Event{ + Op: ReadOp, + FileNum: r.fileNum, + Offset: off, + Size: int64(len(v)), + }) + r.mu.Unlock() + return r.r.ReadAt(ctx, v, off) +} + +// Close is part of the objstorage.Readable interface. +func (r *readable) Close() error { + r.mu.g.flush() + return r.r.Close() +} + +// Size is part of the objstorage.Readable interface. +func (r *readable) Size() int64 { + return r.r.Size() +} + +// NewReadHandle is part of the objstorage.Readable interface. +func (r *readable) NewReadHandle(ctx context.Context) objstorage.ReadHandle { + // It's safe to get the tracer from the generator without the mutex since it never changes. + t := r.mu.g.t + return &readHandle{ + rh: r.r.NewReadHandle(ctx), + fileNum: r.fileNum, + handleID: t.handleID.Add(1), + g: makeEventGenerator(ctx, t), + } +} + +type readHandle struct { + rh objstorage.ReadHandle + fileNum base.FileNum + handleID uint64 + g eventGenerator +} + +var _ objstorage.ReadHandle = (*readHandle)(nil) + +// ReadAt is part of the objstorage.ReadHandle interface. +func (rh *readHandle) ReadAt(ctx context.Context, p []byte, off int64) (n int, err error) { + rh.g.add(ctx, Event{ + Op: ReadOp, + FileNum: rh.fileNum, + HandleID: rh.handleID, + Offset: off, + Size: int64(len(p)), + }) + return rh.rh.ReadAt(ctx, p, off) +} + +// Close is part of the objstorage.ReadHandle interface. +func (rh *readHandle) Close() error { + rh.g.flush() + return rh.rh.Close() +} + +// SetupForCompaction is part of the objstorage.ReadHandle interface. +func (rh *readHandle) SetupForCompaction() { + rh.g.add(context.Background(), Event{ + Op: SetupForCompactionOp, + FileNum: rh.fileNum, + HandleID: rh.handleID, + }) + rh.rh.SetupForCompaction() +} + +// RecordCacheHit is part of the objstorage.ReadHandle interface. +func (rh *readHandle) RecordCacheHit(ctx context.Context, offset, size int64) { + rh.g.add(ctx, Event{ + Op: RecordCacheHitOp, + FileNum: rh.fileNum, + HandleID: rh.handleID, + Offset: offset, + Size: size, + }) + rh.rh.RecordCacheHit(ctx, offset, size) +} + +type ctxInfo struct { + reason Reason + blockType BlockType + levelPlusOne uint8 +} + +func mergeCtxInfo(base, other ctxInfo) ctxInfo { + res := other + if res.reason == 0 { + res.reason = base.reason + } + if res.blockType == 0 { + res.blockType = base.blockType + } + if res.levelPlusOne == 0 { + res.levelPlusOne = base.levelPlusOne + } + return res +} + +type ctxInfoKey struct{} + +func withInfo(ctx context.Context, info ctxInfo) context.Context { + return context.WithValue(ctx, ctxInfoKey{}, info) +} + +func infoFromCtx(ctx context.Context) ctxInfo { + res := ctx.Value(ctxInfoKey{}) + if res == nil { + return ctxInfo{} + } + return res.(ctxInfo) +} + +// WithReason creates a context that has an associated Reason (which ends up in +// traces created under that context). +func WithReason(ctx context.Context, reason Reason) context.Context { + info := infoFromCtx(ctx) + info.reason = reason + return withInfo(ctx, info) +} + +// WithBlockType creates a context that has an associated BlockType (which ends up in +// traces created under that context). +func WithBlockType(ctx context.Context, blockType BlockType) context.Context { + info := infoFromCtx(ctx) + info.blockType = blockType + return withInfo(ctx, info) +} + +// WithLevel creates a context that has an associated level (which ends up in +// traces created under that context). +func WithLevel(ctx context.Context, level int) context.Context { + info := infoFromCtx(ctx) + info.levelPlusOne = uint8(level) + 1 + return withInfo(ctx, info) +} + +const ( + eventSize = int(unsafe.Sizeof(Event{})) + targetEntriesPerFile = 256 * 1024 * 1024 / eventSize // 256MB files + eventsPerBuf = 16 + channelBufSize = 512 * 1024 / eventsPerBuf // 512K events. + bytesPerFileSync = 128 * 1024 +) + +type eventBuf struct { + events [eventsPerBuf]Event + num int +} + +type eventGenerator struct { + t *Tracer + baseCtxInfo ctxInfo + buf eventBuf +} + +func makeEventGenerator(ctx context.Context, t *Tracer) eventGenerator { + return eventGenerator{ + t: t, + baseCtxInfo: infoFromCtx(ctx), + } +} + +func (g *eventGenerator) flush() { + if g.buf.num > 0 { + g.t.workerDataCh <- g.buf + g.buf.num = 0 + } +} + +func (g *eventGenerator) add(ctx context.Context, e Event) { + e.StartUnixNano = time.Now().UnixNano() + info := infoFromCtx(ctx) + info = mergeCtxInfo(g.baseCtxInfo, info) + e.Reason = info.reason + e.BlockType = info.blockType + e.LevelPlusOne = info.levelPlusOne + if g.buf.num == eventsPerBuf { + g.flush() + } + g.buf.events[g.buf.num] = e + g.buf.num++ +} + +type workerState struct { + curFile vfs.File + curBW *bufio.Writer + numEntriesInFile int +} + +func (t *Tracer) workerLoop() { + defer t.workerWait.Done() + stopCh := t.workerStopCh + dataCh := t.workerDataCh + var state workerState + t.workerNewFile(&state) + for { + select { + case <-stopCh: + close(dataCh) + // Flush any remaining traces. + for data := range dataCh { + t.workerWriteTraces(&state, data) + } + t.workerCloseFile(&state) + return + + case data := <-dataCh: + t.workerWriteTraces(&state, data) + } + } +} + +func (t *Tracer) workerWriteTraces(state *workerState, data eventBuf) { + if state.numEntriesInFile >= targetEntriesPerFile { + t.workerCloseFile(state) + t.workerNewFile(state) + } + state.numEntriesInFile += data.num + p := unsafe.Pointer(&data.events[0]) + b := unsafe.Slice((*byte)(p), eventSize*data.num) + if _, err := state.curBW.Write(b); err != nil { + panic(err) + } +} + +func (t *Tracer) workerNewFile(state *workerState) { + filename := fmt.Sprintf("IOTRACES-%s", time.Now().UTC().Format(time.RFC3339Nano)) + + file, err := t.fs.Create(t.fs.PathJoin(t.fsDir, filename)) + if err != nil { + panic(err) + } + file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{ + BytesPerSync: bytesPerFileSync, + }) + state.curFile = file + state.curBW = bufio.NewWriter(file) + state.numEntriesInFile = 0 +} + +func (t *Tracer) workerCloseFile(state *workerState) { + if state.curFile != nil { + if err := state.curBW.Flush(); err != nil { + panic(err) + } + if err := state.curFile.Sync(); err != nil { + panic(err) + } + if err := state.curFile.Close(); err != nil { + panic(err) + } + state.curFile = nil + state.curBW = nil + } +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go new file mode 100644 index 0000000..d0f7450 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go @@ -0,0 +1,522 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "context" + "io" + "os" + "sort" + "sync" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/errors/oserror" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache" + "github.com/cockroachdb/pebble/objstorage/remote" + "github.com/cockroachdb/pebble/vfs" +) + +// provider is the implementation of objstorage.Provider. +type provider struct { + st Settings + + fsDir vfs.File + + tracer *objiotracing.Tracer + + remote remoteSubsystem + + mu struct { + sync.RWMutex + + remote struct { + // catalogBatch accumulates remote object creations and deletions until + // Sync is called. + catalogBatch remoteobjcat.Batch + + storageObjects map[remote.Locator]remote.Storage + } + + // localObjectsChanged is set if non-remote objects were created or deleted + // but Sync was not yet called. + localObjectsChanged bool + + // knownObjects maintains information about objects that are known to the provider. + // It is initialized with the list of files in the manifest when we open a DB. + knownObjects map[base.DiskFileNum]objstorage.ObjectMetadata + + // protectedObjects are objects that cannot be unreferenced because they + // have outstanding SharedObjectBackingHandles. The value is a count of outstanding handles + protectedObjects map[base.DiskFileNum]int + } +} + +var _ objstorage.Provider = (*provider)(nil) + +// Settings that must be specified when creating the provider. +type Settings struct { + Logger base.Logger + + // Local filesystem configuration. + FS vfs.FS + FSDirName string + + // FSDirInitialListing is a listing of FSDirName at the time of calling Open. + // + // This is an optional optimization to avoid double listing on Open when the + // higher layer already has a listing. When nil, we obtain the listing on + // Open. + FSDirInitialListing []string + + // Cleaner cleans obsolete files from the local filesystem. + // + // The default cleaner uses the DeleteCleaner. + FSCleaner base.Cleaner + + // NoSyncOnClose decides whether the implementation will enforce a + // close-time synchronization (e.g., fdatasync() or sync_file_range()) + // on files it writes to. Setting this to true removes the guarantee for a + // sync on close. Some implementations can still issue a non-blocking sync. + NoSyncOnClose bool + + // BytesPerSync enables periodic syncing of files in order to smooth out + // writes to disk. This option does not provide any persistence guarantee, but + // is used to avoid latency spikes if the OS automatically decides to write + // out a large chunk of dirty filesystem buffers. + BytesPerSync int + + // Fields here are set only if the provider is to support remote objects + // (experimental). + Remote struct { + StorageFactory remote.StorageFactory + + // If CreateOnShared is non-zero, sstables are created on remote storage using + // the CreateOnSharedLocator (when the PreferSharedStorage create option is + // true). + CreateOnShared remote.CreateOnSharedStrategy + CreateOnSharedLocator remote.Locator + + // CacheSizeBytes is the size of the on-disk block cache for objects + // on remote storage. If it is 0, no cache is used. + CacheSizeBytes int64 + + // CacheBlockSize is the block size of the cache; if 0, the default of 32KB is used. + CacheBlockSize int + + // ShardingBlockSize is the size of a shard block. The cache is split into contiguous + // ShardingBlockSize units. The units are distributed across multiple independent shards + // of the cache, via a hash(offset) modulo num shards operation. The cache replacement + // policies operate at the level of shard, not whole cache. This is done to reduce lock + // contention. + // + // If ShardingBlockSize is 0, the default of 1 MB is used. + ShardingBlockSize int64 + + // The number of independent shards the cache leverages. Each shard is the same size, + // and a hash of filenum & offset map a read to a certain shard. If set to 0, + // 2*runtime.GOMAXPROCS is used as the shard count. + CacheShardCount int + + // TODO(radu): allow the cache to live on another FS/location (e.g. to use + // instance-local SSD). + } +} + +// DefaultSettings initializes default settings (with no remote storage), +// suitable for tests and tools. +func DefaultSettings(fs vfs.FS, dirName string) Settings { + return Settings{ + Logger: base.DefaultLogger, + FS: fs, + FSDirName: dirName, + FSCleaner: base.DeleteCleaner{}, + NoSyncOnClose: false, + BytesPerSync: 512 * 1024, // 512KB + } +} + +// Open creates the provider. +func Open(settings Settings) (objstorage.Provider, error) { + // Note: we can't just `return open(settings)` because in an error case we + // would return (*provider)(nil) which is not objstorage.Provider(nil). + p, err := open(settings) + if err != nil { + return nil, err + } + return p, nil +} + +func open(settings Settings) (p *provider, _ error) { + fsDir, err := settings.FS.OpenDir(settings.FSDirName) + if err != nil { + return nil, err + } + + defer func() { + if p == nil { + fsDir.Close() + } + }() + + p = &provider{ + st: settings, + fsDir: fsDir, + } + p.mu.knownObjects = make(map[base.DiskFileNum]objstorage.ObjectMetadata) + p.mu.protectedObjects = make(map[base.DiskFileNum]int) + + if objiotracing.Enabled { + p.tracer = objiotracing.Open(settings.FS, settings.FSDirName) + } + + // Add local FS objects. + if err := p.vfsInit(); err != nil { + return nil, err + } + + // Initialize remote subsystem (if configured) and add remote objects. + if err := p.remoteInit(); err != nil { + return nil, err + } + + return p, nil +} + +// Close is part of the objstorage.Provider interface. +func (p *provider) Close() error { + err := p.sharedClose() + if p.fsDir != nil { + err = firstError(err, p.fsDir.Close()) + p.fsDir = nil + } + if objiotracing.Enabled { + if p.tracer != nil { + p.tracer.Close() + p.tracer = nil + } + } + return err +} + +// OpenForReading opens an existing object. +func (p *provider) OpenForReading( + ctx context.Context, + fileType base.FileType, + fileNum base.DiskFileNum, + opts objstorage.OpenOptions, +) (objstorage.Readable, error) { + meta, err := p.Lookup(fileType, fileNum) + if err != nil { + if opts.MustExist { + p.st.Logger.Fatalf("%v", err) + } + return nil, err + } + + var r objstorage.Readable + if !meta.IsRemote() { + r, err = p.vfsOpenForReading(ctx, fileType, fileNum, opts) + } else { + r, err = p.remoteOpenForReading(ctx, meta, opts) + if err != nil && p.isNotExistError(meta, err) { + // Wrap the error so that IsNotExistError functions properly. + err = errors.Mark(err, os.ErrNotExist) + } + } + if err != nil { + return nil, err + } + if objiotracing.Enabled { + r = p.tracer.WrapReadable(ctx, r, fileNum) + } + return r, nil +} + +// Create creates a new object and opens it for writing. +// +// The object is not guaranteed to be durable (accessible in case of crashes) +// until Sync is called. +func (p *provider) Create( + ctx context.Context, + fileType base.FileType, + fileNum base.DiskFileNum, + opts objstorage.CreateOptions, +) (w objstorage.Writable, meta objstorage.ObjectMetadata, err error) { + if opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone { + w, meta, err = p.sharedCreate(ctx, fileType, fileNum, p.st.Remote.CreateOnSharedLocator, opts) + } else { + w, meta, err = p.vfsCreate(ctx, fileType, fileNum) + } + if err != nil { + err = errors.Wrapf(err, "creating object %s", errors.Safe(fileNum)) + return nil, objstorage.ObjectMetadata{}, err + } + p.addMetadata(meta) + if objiotracing.Enabled { + w = p.tracer.WrapWritable(ctx, w, fileNum) + } + return w, meta, nil +} + +// Remove removes an object. +// +// Note that if the object is remote, the object is only (conceptually) removed +// from this provider. If other providers have references on the remote object, +// it will not be removed. +// +// The object is not guaranteed to be durably removed until Sync is called. +func (p *provider) Remove(fileType base.FileType, fileNum base.DiskFileNum) error { + meta, err := p.Lookup(fileType, fileNum) + if err != nil { + return err + } + + if !meta.IsRemote() { + err = p.vfsRemove(fileType, fileNum) + } else { + // TODO(radu): implement remote object removal (i.e. deref). + err = p.sharedUnref(meta) + if err != nil && p.isNotExistError(meta, err) { + // Wrap the error so that IsNotExistError functions properly. + err = errors.Mark(err, os.ErrNotExist) + } + } + if err != nil && !p.IsNotExistError(err) { + // We want to be able to retry a Remove, so we keep the object in our list. + // TODO(radu): we should mark the object as "zombie" and not allow any other + // operations. + return errors.Wrapf(err, "removing object %s", errors.Safe(fileNum)) + } + + p.removeMetadata(fileNum) + return err +} + +func (p *provider) isNotExistError(meta objstorage.ObjectMetadata, err error) bool { + if meta.Remote.Storage != nil { + return meta.Remote.Storage.IsNotExistError(err) + } + return oserror.IsNotExist(err) +} + +// IsNotExistError is part of the objstorage.Provider interface. +func (p *provider) IsNotExistError(err error) bool { + // We use errors.Mark(err, os.ErrNotExist) for not-exist errors coming from + // remote.Storage. + return oserror.IsNotExist(err) +} + +// Sync flushes the metadata from creation or removal of objects since the last Sync. +func (p *provider) Sync() error { + if err := p.vfsSync(); err != nil { + return err + } + if err := p.sharedSync(); err != nil { + return err + } + return nil +} + +// LinkOrCopyFromLocal creates a new object that is either a copy of a given +// local file or a hard link (if the new object is created on the same FS, and +// if the FS supports it). +// +// The object is not guaranteed to be durable (accessible in case of crashes) +// until Sync is called. +func (p *provider) LinkOrCopyFromLocal( + ctx context.Context, + srcFS vfs.FS, + srcFilePath string, + dstFileType base.FileType, + dstFileNum base.DiskFileNum, + opts objstorage.CreateOptions, +) (objstorage.ObjectMetadata, error) { + shared := opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone + if !shared && srcFS == p.st.FS { + // Wrap the normal filesystem with one which wraps newly created files with + // vfs.NewSyncingFile. + fs := vfs.NewSyncingFS(p.st.FS, vfs.SyncingFileOptions{ + NoSyncOnClose: p.st.NoSyncOnClose, + BytesPerSync: p.st.BytesPerSync, + }) + dstPath := p.vfsPath(dstFileType, dstFileNum) + if err := vfs.LinkOrCopy(fs, srcFilePath, dstPath); err != nil { + return objstorage.ObjectMetadata{}, err + } + + meta := objstorage.ObjectMetadata{ + DiskFileNum: dstFileNum, + FileType: dstFileType, + } + p.addMetadata(meta) + return meta, nil + } + // Create the object and copy the data. + w, meta, err := p.Create(ctx, dstFileType, dstFileNum, opts) + if err != nil { + return objstorage.ObjectMetadata{}, err + } + f, err := srcFS.Open(srcFilePath, vfs.SequentialReadsOption) + if err != nil { + return objstorage.ObjectMetadata{}, err + } + defer f.Close() + buf := make([]byte, 64*1024) + for { + n, readErr := f.Read(buf) + if readErr != nil && readErr != io.EOF { + w.Abort() + return objstorage.ObjectMetadata{}, readErr + } + + if n > 0 { + if err := w.Write(buf[:n]); err != nil { + w.Abort() + return objstorage.ObjectMetadata{}, err + } + } + + if readErr == io.EOF { + break + } + } + if err := w.Finish(); err != nil { + return objstorage.ObjectMetadata{}, err + } + return meta, nil +} + +// Lookup is part of the objstorage.Provider interface. +func (p *provider) Lookup( + fileType base.FileType, fileNum base.DiskFileNum, +) (objstorage.ObjectMetadata, error) { + p.mu.RLock() + defer p.mu.RUnlock() + meta, ok := p.mu.knownObjects[fileNum] + if !ok { + return objstorage.ObjectMetadata{}, errors.Wrapf( + os.ErrNotExist, + "file %s (type %d) unknown to the objstorage provider", + errors.Safe(fileNum), errors.Safe(fileType), + ) + } + if meta.FileType != fileType { + return objstorage.ObjectMetadata{}, errors.AssertionFailedf( + "file %s type mismatch (known type %d, expected type %d)", + errors.Safe(fileNum), errors.Safe(meta.FileType), errors.Safe(fileType), + ) + } + return meta, nil +} + +// Path is part of the objstorage.Provider interface. +func (p *provider) Path(meta objstorage.ObjectMetadata) string { + if !meta.IsRemote() { + return p.vfsPath(meta.FileType, meta.DiskFileNum) + } + return p.remotePath(meta) +} + +// Size returns the size of the object. +func (p *provider) Size(meta objstorage.ObjectMetadata) (int64, error) { + if !meta.IsRemote() { + return p.vfsSize(meta.FileType, meta.DiskFileNum) + } + return p.remoteSize(meta) +} + +// List is part of the objstorage.Provider interface. +func (p *provider) List() []objstorage.ObjectMetadata { + p.mu.RLock() + defer p.mu.RUnlock() + res := make([]objstorage.ObjectMetadata, 0, len(p.mu.knownObjects)) + for _, meta := range p.mu.knownObjects { + res = append(res, meta) + } + sort.Slice(res, func(i, j int) bool { + return res[i].DiskFileNum.FileNum() < res[j].DiskFileNum.FileNum() + }) + return res +} + +// Metrics is part of the objstorage.Provider interface. +func (p *provider) Metrics() sharedcache.Metrics { + if p.remote.cache != nil { + return p.remote.cache.Metrics() + } + return sharedcache.Metrics{} +} + +func (p *provider) addMetadata(meta objstorage.ObjectMetadata) { + if invariants.Enabled { + meta.AssertValid() + } + p.mu.Lock() + defer p.mu.Unlock() + p.mu.knownObjects[meta.DiskFileNum] = meta + if meta.IsRemote() { + p.mu.remote.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{ + FileNum: meta.DiskFileNum, + FileType: meta.FileType, + CreatorID: meta.Remote.CreatorID, + CreatorFileNum: meta.Remote.CreatorFileNum, + Locator: meta.Remote.Locator, + CleanupMethod: meta.Remote.CleanupMethod, + }) + } else { + p.mu.localObjectsChanged = true + } +} + +func (p *provider) removeMetadata(fileNum base.DiskFileNum) { + p.mu.Lock() + defer p.mu.Unlock() + + meta, ok := p.mu.knownObjects[fileNum] + if !ok { + return + } + delete(p.mu.knownObjects, fileNum) + if meta.IsRemote() { + p.mu.remote.catalogBatch.DeleteObject(fileNum) + } else { + p.mu.localObjectsChanged = true + } +} + +// protectObject prevents the unreferencing of a remote object until +// unprotectObject is called. +func (p *provider) protectObject(fileNum base.DiskFileNum) { + p.mu.Lock() + defer p.mu.Unlock() + p.mu.protectedObjects[fileNum] = p.mu.protectedObjects[fileNum] + 1 +} + +func (p *provider) unprotectObject(fileNum base.DiskFileNum) { + p.mu.Lock() + defer p.mu.Unlock() + v := p.mu.protectedObjects[fileNum] + if invariants.Enabled && v == 0 { + panic("invalid protection count") + } + if v > 1 { + p.mu.protectedObjects[fileNum] = v - 1 + } else { + delete(p.mu.protectedObjects, fileNum) + // TODO(radu): check if the object is still in knownObject; if not, unref it + // now. + } +} + +func (p *provider) isProtected(fileNum base.DiskFileNum) bool { + p.mu.Lock() + defer p.mu.Unlock() + return p.mu.protectedObjects[fileNum] > 0 +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/readahead.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go similarity index 80% rename from vendor/github.com/cockroachdb/pebble/objstorage/readahead.go rename to vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go index 38984c5..e07017c 100644 --- a/vendor/github.com/cockroachdb/pebble/objstorage/readahead.go +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go @@ -2,7 +2,7 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -package objstorage +package objstorageprovider const ( // Constants for dynamic readahead of data blocks. Note that the size values @@ -11,15 +11,15 @@ const ( minFileReadsForReadahead = 2 // TODO(bilal): Have the initial size value be a factor of the block size, // as opposed to a hardcoded value. - initialReadaheadSize = 64 << 10 /* 64KB */ - maxReadaheadSize = 256 << 10 /* 256KB */ + initialReadaheadSize = 64 << 10 /* 64KB */ ) // readaheadState contains state variables related to readahead. Updated on // file reads. type readaheadState struct { // Number of sequential reads. - numReads int64 + numReads int64 + maxReadaheadSize int64 // Size issued to the next call to Prefetch. Starts at or above // initialReadaheadSize and grows exponentially until maxReadaheadSize. size int64 @@ -32,22 +32,23 @@ type readaheadState struct { limit int64 } -func makeReadaheadState() readaheadState { +func makeReadaheadState(maxReadaheadSize int64) readaheadState { return readaheadState{ - size: initialReadaheadSize, + size: initialReadaheadSize, + maxReadaheadSize: maxReadaheadSize, } } func (rs *readaheadState) recordCacheHit(offset, blockLength int64) { currentReadEnd := offset + blockLength if rs.numReads >= minFileReadsForReadahead { - if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { + if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize { // This is a read that would have resulted in a readahead, had it // not been a cache hit. rs.limit = currentReadEnd return } - if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize { + if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+rs.maxReadaheadSize { // We read too far away from rs.limit to benefit from readahead in // any scenario. Reset all variables. rs.numReads = 1 @@ -61,7 +62,7 @@ func (rs *readaheadState) recordCacheHit(offset, blockLength int64) { // readahead. return } - if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { + if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize { // Blocks are being read sequentially and would benefit from readahead // down the line. rs.numReads++ @@ -89,28 +90,28 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 { // as well as the interval where a read would benefit from read ahead: // [rs.limit, rs.limit + rs.size] // We increase the latter interval to - // [rs.limit, rs.limit + maxReadaheadSize] to account for cases where + // [rs.limit, rs.limit + rs.maxReadaheadSize] to account for cases where // readahead may not be beneficial with a small readahead size, but over // time the readahead size would increase exponentially to make it // beneficial. - if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { + if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize { // We are doing a read in the interval ahead of // the last readahead range. In the diagrams below, ++++ is the last // readahead range, ==== is the range represented by - // [rs.limit, rs.limit + maxReadaheadSize], and ---- is the range + // [rs.limit, rs.limit + rs.maxReadaheadSize], and ---- is the range // being read. // - // rs.limit rs.limit + maxReadaheadSize + // rs.limit rs.limit + rs.maxReadaheadSize // ++++++++++|===========================| // // |-------------| // offset currentReadEnd // // This case is also possible, as are all cases with an overlap - // between [rs.limit, rs.limit + maxReadaheadSize] and [offset, + // between [rs.limit, rs.limit + rs.maxReadaheadSize] and [offset, // currentReadEnd]: // - // rs.limit rs.limit + maxReadaheadSize + // rs.limit rs.limit + rs.maxReadaheadSize // ++++++++++|===========================| // // |-------------| @@ -122,19 +123,19 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 { rs.prevSize = rs.size // Increase rs.size for the next read. rs.size *= 2 - if rs.size > maxReadaheadSize { - rs.size = maxReadaheadSize + if rs.size > rs.maxReadaheadSize { + rs.size = rs.maxReadaheadSize } return rs.prevSize } - if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize { + if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+rs.maxReadaheadSize { // The above conditional has rs.limit > rs.prevSize to confirm that // rs.limit - rs.prevSize would not underflow. // We read too far away from rs.limit to benefit from readahead in // any scenario. Reset all variables. // The case where we read too far ahead: // - // (rs.limit - rs.prevSize) (rs.limit) (rs.limit + maxReadaheadSize) + // (rs.limit - rs.prevSize) (rs.limit) (rs.limit + rs.maxReadaheadSize) // |+++++++++++++|=============| // // |-------------| @@ -142,7 +143,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 { // // Or too far behind: // - // (rs.limit - rs.prevSize) (rs.limit) (rs.limit + maxReadaheadSize) + // (rs.limit - rs.prevSize) (rs.limit) (rs.limit + rs.maxReadaheadSize) // |+++++++++++++|=============| // // |-------------| @@ -160,7 +161,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 { // readahead, but there's no reason to issue a readahead call at the // moment. // - // (rs.limit - rs.prevSize) (rs.limit + maxReadaheadSize) + // (rs.limit - rs.prevSize) (rs.limit + rs.maxReadaheadSize) // |+++++++++++++|===============| // (rs.limit) // @@ -170,11 +171,11 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 { rs.numReads++ return 0 } - if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { + if currentReadEnd >= rs.limit && offset <= rs.limit+rs.maxReadaheadSize { // Blocks are being read sequentially and would benefit from readahead // down the line. // - // (rs.limit) (rs.limit + maxReadaheadSize) + // (rs.limit) (rs.limit + rs.maxReadaheadSize) // |=============| // // |-------| @@ -186,7 +187,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 { // We read too far ahead of the last read, or before it. This indicates // a random read, where readahead is not desirable. Reset all variables. // - // (rs.limit - maxReadaheadSize) (rs.limit) (rs.limit + maxReadaheadSize) + // (rs.limit - rs.maxReadaheadSize) (rs.limit) (rs.limit + rs.maxReadaheadSize) // |+++++++++++++|=============| // // |-------| diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote.go new file mode 100644 index 0000000..e8ee42b --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote.go @@ -0,0 +1,376 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "context" + "fmt" + "runtime" + "sync" + "sync/atomic" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache" + "github.com/cockroachdb/pebble/objstorage/remote" +) + +// remoteSubsystem contains the provider fields related to remote storage. +// All fields remain unset if remote storage is not configured. +type remoteSubsystem struct { + catalog *remoteobjcat.Catalog + // catalogSyncMutex is used to correctly serialize two sharedSync operations. + // It must be acquired before the provider mutex. + catalogSyncMutex sync.Mutex + + cache *sharedcache.Cache + + // shared contains the fields relevant to shared objects, i.e. objects that + // are created by Pebble and potentially shared between Pebble instances. + shared struct { + // initialized guards access to the creatorID field. + initialized atomic.Bool + creatorID objstorage.CreatorID + initOnce sync.Once + + // checkRefsOnOpen controls whether we check the ref marker file when opening + // an object. Normally this is true when invariants are enabled (but the provider + // test tweaks this field). + checkRefsOnOpen bool + } +} + +// remoteInit initializes the remote object subsystem (if configured) and finds +// any remote objects. +func (p *provider) remoteInit() error { + if p.st.Remote.StorageFactory == nil { + return nil + } + catalog, contents, err := remoteobjcat.Open(p.st.FS, p.st.FSDirName) + if err != nil { + return errors.Wrapf(err, "pebble: could not open remote object catalog") + } + p.remote.catalog = catalog + p.remote.shared.checkRefsOnOpen = invariants.Enabled + + // The creator ID may or may not be initialized yet. + if contents.CreatorID.IsSet() { + p.remote.initShared(contents.CreatorID) + p.st.Logger.Infof("remote storage configured; creatorID = %s", contents.CreatorID) + } else { + p.st.Logger.Infof("remote storage configured; no creatorID yet") + } + + if p.st.Remote.CacheSizeBytes > 0 { + const defaultBlockSize = 32 * 1024 + blockSize := p.st.Remote.CacheBlockSize + if blockSize == 0 { + blockSize = defaultBlockSize + } + + const defaultShardingBlockSize = 1024 * 1024 + shardingBlockSize := p.st.Remote.ShardingBlockSize + if shardingBlockSize == 0 { + shardingBlockSize = defaultShardingBlockSize + } + + numShards := p.st.Remote.CacheShardCount + if numShards == 0 { + numShards = 2 * runtime.GOMAXPROCS(0) + } + + p.remote.cache, err = sharedcache.Open( + p.st.FS, p.st.Logger, p.st.FSDirName, blockSize, shardingBlockSize, p.st.Remote.CacheSizeBytes, numShards) + if err != nil { + return errors.Wrapf(err, "pebble: could not open remote object cache") + } + } + + for _, meta := range contents.Objects { + o := objstorage.ObjectMetadata{ + DiskFileNum: meta.FileNum, + FileType: meta.FileType, + } + o.Remote.CreatorID = meta.CreatorID + o.Remote.CreatorFileNum = meta.CreatorFileNum + o.Remote.CleanupMethod = meta.CleanupMethod + o.Remote.Locator = meta.Locator + o.Remote.CustomObjectName = meta.CustomObjectName + o.Remote.Storage, err = p.ensureStorageLocked(o.Remote.Locator) + if err != nil { + return errors.Wrapf(err, "creating remote.Storage object for locator '%s'", o.Remote.Locator) + } + if invariants.Enabled { + o.AssertValid() + } + p.mu.knownObjects[o.DiskFileNum] = o + } + return nil +} + +// initShared initializes the creator ID, allowing use of shared objects. +func (ss *remoteSubsystem) initShared(creatorID objstorage.CreatorID) { + ss.shared.initOnce.Do(func() { + ss.shared.creatorID = creatorID + ss.shared.initialized.Store(true) + }) +} + +func (p *provider) sharedClose() error { + if p.st.Remote.StorageFactory == nil { + return nil + } + var err error + if p.remote.cache != nil { + err = p.remote.cache.Close() + p.remote.cache = nil + } + if p.remote.catalog != nil { + err = firstError(err, p.remote.catalog.Close()) + p.remote.catalog = nil + } + return err +} + +// SetCreatorID is part of the objstorage.Provider interface. +func (p *provider) SetCreatorID(creatorID objstorage.CreatorID) error { + if p.st.Remote.StorageFactory == nil { + return errors.AssertionFailedf("attempt to set CreatorID but remote storage not enabled") + } + // Note: this call is a cheap no-op if the creator ID was already set. This + // call also checks if we are trying to change the ID. + if err := p.remote.catalog.SetCreatorID(creatorID); err != nil { + return err + } + if !p.remote.shared.initialized.Load() { + p.st.Logger.Infof("remote storage creatorID set to %s", creatorID) + p.remote.initShared(creatorID) + } + return nil +} + +// IsSharedForeign is part of the objstorage.Provider interface. +func (p *provider) IsSharedForeign(meta objstorage.ObjectMetadata) bool { + if !p.remote.shared.initialized.Load() { + return false + } + return meta.IsShared() && (meta.Remote.CreatorID != p.remote.shared.creatorID) +} + +func (p *provider) remoteCheckInitialized() error { + if p.st.Remote.StorageFactory == nil { + return errors.Errorf("remote object support not configured") + } + return nil +} + +func (p *provider) sharedCheckInitialized() error { + if err := p.remoteCheckInitialized(); err != nil { + return err + } + if !p.remote.shared.initialized.Load() { + return errors.Errorf("remote object support not available: remote creator ID not yet set") + } + return nil +} + +func (p *provider) sharedSync() error { + // Serialize parallel sync operations. Note that ApplyBatch is already + // serialized internally, but we want to make sure they get called with + // batches in the right order. + p.remote.catalogSyncMutex.Lock() + defer p.remote.catalogSyncMutex.Unlock() + + batch := func() remoteobjcat.Batch { + p.mu.Lock() + defer p.mu.Unlock() + res := p.mu.remote.catalogBatch.Copy() + p.mu.remote.catalogBatch.Reset() + return res + }() + + if batch.IsEmpty() { + return nil + } + + if err := p.remote.catalog.ApplyBatch(batch); err != nil { + // Put back the batch (for the next Sync), appending any operations that + // happened in the meantime. + p.mu.Lock() + defer p.mu.Unlock() + batch.Append(p.mu.remote.catalogBatch) + p.mu.remote.catalogBatch = batch + return err + } + + return nil +} + +func (p *provider) remotePath(meta objstorage.ObjectMetadata) string { + if meta.Remote.Locator != "" { + return fmt.Sprintf("remote-%s://%s", meta.Remote.Locator, remoteObjectName(meta)) + } + return "remote://" + remoteObjectName(meta) +} + +// sharedCreateRef creates a reference marker object. +func (p *provider) sharedCreateRef(meta objstorage.ObjectMetadata) error { + if err := p.sharedCheckInitialized(); err != nil { + return err + } + if meta.Remote.CleanupMethod != objstorage.SharedRefTracking { + return nil + } + refName := p.sharedObjectRefName(meta) + writer, err := meta.Remote.Storage.CreateObject(refName) + if err == nil { + // The object is empty, just close the writer. + err = writer.Close() + } + if err != nil { + return errors.Wrapf(err, "creating marker object %q", refName) + } + return nil +} + +func (p *provider) sharedCreate( + _ context.Context, + fileType base.FileType, + fileNum base.DiskFileNum, + locator remote.Locator, + opts objstorage.CreateOptions, +) (objstorage.Writable, objstorage.ObjectMetadata, error) { + if err := p.sharedCheckInitialized(); err != nil { + return nil, objstorage.ObjectMetadata{}, err + } + storage, err := p.ensureStorage(locator) + if err != nil { + return nil, objstorage.ObjectMetadata{}, err + } + meta := objstorage.ObjectMetadata{ + DiskFileNum: fileNum, + FileType: fileType, + } + meta.Remote.CreatorID = p.remote.shared.creatorID + meta.Remote.CreatorFileNum = fileNum + meta.Remote.CleanupMethod = opts.SharedCleanupMethod + meta.Remote.Locator = locator + meta.Remote.Storage = storage + + objName := remoteObjectName(meta) + writer, err := storage.CreateObject(objName) + if err != nil { + return nil, objstorage.ObjectMetadata{}, errors.Wrapf(err, "creating object %q", objName) + } + return &sharedWritable{ + p: p, + meta: meta, + storageWriter: writer, + }, meta, nil +} + +func (p *provider) remoteOpenForReading( + ctx context.Context, meta objstorage.ObjectMetadata, opts objstorage.OpenOptions, +) (objstorage.Readable, error) { + if err := p.remoteCheckInitialized(); err != nil { + return nil, err + } + // Verify we have a reference on this object; for performance reasons, we only + // do this in testing scenarios. + if p.remote.shared.checkRefsOnOpen && meta.Remote.CleanupMethod == objstorage.SharedRefTracking { + if err := p.sharedCheckInitialized(); err != nil { + return nil, err + } + refName := p.sharedObjectRefName(meta) + if _, err := meta.Remote.Storage.Size(refName); err != nil { + if meta.Remote.Storage.IsNotExistError(err) { + if opts.MustExist { + p.st.Logger.Fatalf("marker object %q does not exist", refName) + // TODO(radu): maybe list references for the object. + } + return nil, errors.Errorf("marker object %q does not exist", refName) + } + return nil, errors.Wrapf(err, "checking marker object %q", refName) + } + } + objName := remoteObjectName(meta) + reader, size, err := meta.Remote.Storage.ReadObject(ctx, objName) + if err != nil { + if opts.MustExist && meta.Remote.Storage.IsNotExistError(err) { + p.st.Logger.Fatalf("object %q does not exist", objName) + // TODO(radu): maybe list references for the object. + } + return nil, err + } + return p.newRemoteReadable(reader, size, meta.DiskFileNum), nil +} + +func (p *provider) remoteSize(meta objstorage.ObjectMetadata) (int64, error) { + if err := p.remoteCheckInitialized(); err != nil { + return 0, err + } + objName := remoteObjectName(meta) + return meta.Remote.Storage.Size(objName) +} + +// sharedUnref implements object "removal" with the remote backend. The ref +// marker object is removed and the backing object is removed only if there are +// no other ref markers. +func (p *provider) sharedUnref(meta objstorage.ObjectMetadata) error { + if meta.Remote.CleanupMethod == objstorage.SharedNoCleanup { + // Never delete objects in this mode. + return nil + } + if p.isProtected(meta.DiskFileNum) { + // TODO(radu): we need a mechanism to unref the object when it becomes + // unprotected. + return nil + } + + refName := p.sharedObjectRefName(meta) + // Tolerate a not-exists error. + if err := meta.Remote.Storage.Delete(refName); err != nil && !meta.Remote.Storage.IsNotExistError(err) { + return err + } + otherRefs, err := meta.Remote.Storage.List(sharedObjectRefPrefix(meta), "" /* delimiter */) + if err != nil { + return err + } + if len(otherRefs) == 0 { + objName := remoteObjectName(meta) + if err := meta.Remote.Storage.Delete(objName); err != nil && !meta.Remote.Storage.IsNotExistError(err) { + return err + } + } + return nil +} + +// ensureStorageLocked populates the remote.Storage object for the given +// locator, if necessary. p.mu must be held. +func (p *provider) ensureStorageLocked(locator remote.Locator) (remote.Storage, error) { + if p.mu.remote.storageObjects == nil { + p.mu.remote.storageObjects = make(map[remote.Locator]remote.Storage) + } + if res, ok := p.mu.remote.storageObjects[locator]; ok { + return res, nil + } + res, err := p.st.Remote.StorageFactory.CreateStorage(locator) + if err != nil { + return nil, err + } + + p.mu.remote.storageObjects[locator] = res + return res, nil +} + +// ensureStorage populates the remote.Storage object for the given locator, if necessary. +func (p *provider) ensureStorage(locator remote.Locator) (remote.Storage, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.ensureStorageLocked(locator) +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_backing.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_backing.go new file mode 100644 index 0000000..d302151 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_backing.go @@ -0,0 +1,303 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "bytes" + "encoding/binary" + "io" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat" + "github.com/cockroachdb/pebble/objstorage/remote" +) + +const ( + tagCreatorID = 1 + tagCreatorFileNum = 2 + tagCleanupMethod = 3 + // tagRefCheckID encodes the information for a ref marker that needs to be + // checked when attaching this object to another provider. This is set to the + // creator ID and FileNum for the provider that encodes the backing, and + // allows the "target" provider to check that the "source" provider kept its + // reference on the object alive. + tagRefCheckID = 4 + // tagLocator encodes the remote.Locator; if absent the locator is "". It is + // followed by the locator string length and the locator string. + tagLocator = 5 + // tagLocator encodes a custom object name (if present). It is followed by the + // custom name string length and the string. + tagCustomObjectName = 6 + + // Any new tags that don't have the tagNotSafeToIgnoreMask bit set must be + // followed by the length of the data (so they can be skipped). + + // Any new tags that have the tagNotSafeToIgnoreMask bit set cause errors if + // they are encountered by earlier code that doesn't know the tag. + tagNotSafeToIgnoreMask = 64 +) + +func (p *provider) encodeRemoteObjectBacking( + meta *objstorage.ObjectMetadata, +) (objstorage.RemoteObjectBacking, error) { + if !meta.IsRemote() { + return nil, errors.AssertionFailedf("object %s not on remote storage", meta.DiskFileNum) + } + + buf := make([]byte, 0, binary.MaxVarintLen64*4) + buf = binary.AppendUvarint(buf, tagCreatorID) + buf = binary.AppendUvarint(buf, uint64(meta.Remote.CreatorID)) + // TODO(radu): encode file type as well? + buf = binary.AppendUvarint(buf, tagCreatorFileNum) + buf = binary.AppendUvarint(buf, uint64(meta.Remote.CreatorFileNum.FileNum())) + buf = binary.AppendUvarint(buf, tagCleanupMethod) + buf = binary.AppendUvarint(buf, uint64(meta.Remote.CleanupMethod)) + if meta.Remote.CleanupMethod == objstorage.SharedRefTracking { + buf = binary.AppendUvarint(buf, tagRefCheckID) + buf = binary.AppendUvarint(buf, uint64(p.remote.shared.creatorID)) + buf = binary.AppendUvarint(buf, uint64(meta.DiskFileNum.FileNum())) + } + if meta.Remote.Locator != "" { + buf = binary.AppendUvarint(buf, tagLocator) + buf = encodeString(buf, string(meta.Remote.Locator)) + } + if meta.Remote.CustomObjectName != "" { + buf = binary.AppendUvarint(buf, tagCustomObjectName) + buf = encodeString(buf, meta.Remote.CustomObjectName) + } + return buf, nil +} + +type remoteObjectBackingHandle struct { + backing objstorage.RemoteObjectBacking + fileNum base.DiskFileNum + p *provider +} + +func (s *remoteObjectBackingHandle) Get() (objstorage.RemoteObjectBacking, error) { + if s.backing == nil { + return nil, errors.Errorf("RemoteObjectBackingHandle.Get() called after Close()") + } + return s.backing, nil +} + +func (s *remoteObjectBackingHandle) Close() { + if s.backing != nil { + s.backing = nil + s.p.unprotectObject(s.fileNum) + } +} + +var _ objstorage.RemoteObjectBackingHandle = (*remoteObjectBackingHandle)(nil) + +// RemoteObjectBacking is part of the objstorage.Provider interface. +func (p *provider) RemoteObjectBacking( + meta *objstorage.ObjectMetadata, +) (objstorage.RemoteObjectBackingHandle, error) { + backing, err := p.encodeRemoteObjectBacking(meta) + if err != nil { + return nil, err + } + p.protectObject(meta.DiskFileNum) + return &remoteObjectBackingHandle{ + backing: backing, + fileNum: meta.DiskFileNum, + p: p, + }, nil +} + +// CreateExternalObjectBacking is part of the objstorage.Provider interface. +func (p *provider) CreateExternalObjectBacking( + locator remote.Locator, objName string, +) (objstorage.RemoteObjectBacking, error) { + var meta objstorage.ObjectMetadata + meta.Remote.Locator = locator + meta.Remote.CustomObjectName = objName + meta.Remote.CleanupMethod = objstorage.SharedNoCleanup + return p.encodeRemoteObjectBacking(&meta) +} + +type decodedBacking struct { + meta objstorage.ObjectMetadata + // refToCheck is set only when meta.Remote.CleanupMethod is RefTracking + refToCheck struct { + creatorID objstorage.CreatorID + fileNum base.DiskFileNum + } +} + +// decodeRemoteObjectBacking decodes the remote object metadata. +// +// Note that the meta.Remote.Storage field is not set. +func decodeRemoteObjectBacking( + fileType base.FileType, fileNum base.DiskFileNum, buf objstorage.RemoteObjectBacking, +) (decodedBacking, error) { + var creatorID, creatorFileNum, cleanupMethod, refCheckCreatorID, refCheckFileNum uint64 + var locator, customObjName string + br := bytes.NewReader(buf) + for { + tag, err := binary.ReadUvarint(br) + if err == io.EOF { + break + } + if err != nil { + return decodedBacking{}, err + } + switch tag { + case tagCreatorID: + creatorID, err = binary.ReadUvarint(br) + + case tagCreatorFileNum: + creatorFileNum, err = binary.ReadUvarint(br) + + case tagCleanupMethod: + cleanupMethod, err = binary.ReadUvarint(br) + + case tagRefCheckID: + refCheckCreatorID, err = binary.ReadUvarint(br) + if err == nil { + refCheckFileNum, err = binary.ReadUvarint(br) + } + + case tagLocator: + locator, err = decodeString(br) + + case tagCustomObjectName: + customObjName, err = decodeString(br) + + default: + // Ignore unknown tags, unless they're not safe to ignore. + if tag&tagNotSafeToIgnoreMask != 0 { + return decodedBacking{}, errors.Newf("unknown tag %d", tag) + } + var dataLen uint64 + dataLen, err = binary.ReadUvarint(br) + if err == nil { + _, err = br.Seek(int64(dataLen), io.SeekCurrent) + } + } + if err != nil { + return decodedBacking{}, err + } + } + if customObjName == "" { + if creatorID == 0 { + return decodedBacking{}, errors.Newf("remote object backing missing creator ID") + } + if creatorFileNum == 0 { + return decodedBacking{}, errors.Newf("remote object backing missing creator file num") + } + } + var res decodedBacking + res.meta.DiskFileNum = fileNum + res.meta.FileType = fileType + res.meta.Remote.CreatorID = objstorage.CreatorID(creatorID) + res.meta.Remote.CreatorFileNum = base.FileNum(creatorFileNum).DiskFileNum() + res.meta.Remote.CleanupMethod = objstorage.SharedCleanupMethod(cleanupMethod) + if res.meta.Remote.CleanupMethod == objstorage.SharedRefTracking { + if refCheckCreatorID == 0 || refCheckFileNum == 0 { + return decodedBacking{}, errors.Newf("remote object backing missing ref to check") + } + res.refToCheck.creatorID = objstorage.CreatorID(refCheckCreatorID) + res.refToCheck.fileNum = base.FileNum(refCheckFileNum).DiskFileNum() + } + res.meta.Remote.Locator = remote.Locator(locator) + res.meta.Remote.CustomObjectName = customObjName + return res, nil +} + +func encodeString(buf []byte, s string) []byte { + buf = binary.AppendUvarint(buf, uint64(len(s))) + buf = append(buf, []byte(s)...) + return buf +} + +func decodeString(br io.ByteReader) (string, error) { + length, err := binary.ReadUvarint(br) + if err != nil || length == 0 { + return "", err + } + buf := make([]byte, length) + for i := range buf { + buf[i], err = br.ReadByte() + if err != nil { + return "", err + } + } + return string(buf), nil +} + +// AttachRemoteObjects is part of the objstorage.Provider interface. +func (p *provider) AttachRemoteObjects( + objs []objstorage.RemoteObjectToAttach, +) ([]objstorage.ObjectMetadata, error) { + decoded := make([]decodedBacking, len(objs)) + for i, o := range objs { + var err error + decoded[i], err = decodeRemoteObjectBacking(o.FileType, o.FileNum, o.Backing) + if err != nil { + return nil, err + } + decoded[i].meta.Remote.Storage, err = p.ensureStorage(decoded[i].meta.Remote.Locator) + if err != nil { + return nil, err + } + } + + // Create the reference marker objects. + // TODO(radu): parallelize this. + for _, d := range decoded { + if d.meta.Remote.CleanupMethod != objstorage.SharedRefTracking { + continue + } + if err := p.sharedCreateRef(d.meta); err != nil { + // TODO(radu): clean up references previously created in this loop. + return nil, err + } + // Check the "origin"'s reference. + refName := sharedObjectRefName(d.meta, d.refToCheck.creatorID, d.refToCheck.fileNum) + if _, err := d.meta.Remote.Storage.Size(refName); err != nil { + _ = p.sharedUnref(d.meta) + // TODO(radu): clean up references previously created in this loop. + if d.meta.Remote.Storage.IsNotExistError(err) { + return nil, errors.Errorf("origin marker object %q does not exist;"+ + " object probably removed from the provider which created the backing", refName) + } + return nil, errors.Wrapf(err, "checking origin's marker object %s", refName) + } + } + + func() { + p.mu.Lock() + defer p.mu.Unlock() + for _, d := range decoded { + p.mu.remote.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{ + FileNum: d.meta.DiskFileNum, + FileType: d.meta.FileType, + CreatorID: d.meta.Remote.CreatorID, + CreatorFileNum: d.meta.Remote.CreatorFileNum, + CleanupMethod: d.meta.Remote.CleanupMethod, + Locator: d.meta.Remote.Locator, + }) + } + }() + if err := p.sharedSync(); err != nil { + return nil, err + } + + metas := make([]objstorage.ObjectMetadata, len(decoded)) + for i, d := range decoded { + metas[i] = d.meta + } + + p.mu.Lock() + defer p.mu.Unlock() + for _, meta := range metas { + p.mu.knownObjects[meta.DiskFileNum] = meta + } + return metas, nil +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_obj_name.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_obj_name.go new file mode 100644 index 0000000..b33a908 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_obj_name.go @@ -0,0 +1,91 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "fmt" + + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" +) + +// remoteObjectName returns the name of an object on remote storage. +// +// For sstables, the format is: --.sst +// For example: 1a3f-2-000001.sst +func remoteObjectName(meta objstorage.ObjectMetadata) string { + if meta.Remote.CustomObjectName != "" { + return meta.Remote.CustomObjectName + } + switch meta.FileType { + case base.FileTypeTable: + return fmt.Sprintf( + "%04x-%d-%06d.sst", + objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(), + ) + } + panic("unknown FileType") +} + +// sharedObjectRefName returns the name of the object's ref marker associated +// with a given referencing provider. This name is the object's name concatenated with +// ".ref..". +// +// For example: 1a3f-2-000001.sst.ref.5.000008 +func sharedObjectRefName( + meta objstorage.ObjectMetadata, refCreatorID objstorage.CreatorID, refFileNum base.DiskFileNum, +) string { + if meta.Remote.CleanupMethod != objstorage.SharedRefTracking { + panic("ref object used when ref tracking disabled") + } + if meta.Remote.CustomObjectName != "" { + return fmt.Sprintf( + "%s.ref.%d.%06d", meta.Remote.CustomObjectName, refCreatorID, refFileNum.FileNum(), + ) + } + switch meta.FileType { + case base.FileTypeTable: + return fmt.Sprintf( + "%04x-%d-%06d.sst.ref.%d.%06d", + objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(), refCreatorID, refFileNum.FileNum(), + ) + } + panic("unknown FileType") +} + +func sharedObjectRefPrefix(meta objstorage.ObjectMetadata) string { + if meta.Remote.CustomObjectName != "" { + return meta.Remote.CustomObjectName + ".ref." + } + switch meta.FileType { + case base.FileTypeTable: + return fmt.Sprintf( + "%04x-%d-%06d.sst.ref.", + objHash(meta), meta.Remote.CreatorID, meta.Remote.CreatorFileNum.FileNum(), + ) + } + panic("unknown FileType") +} + +// sharedObjectRefName returns the name of the object's ref marker associated +// with this provider. This name is the object's name concatenated with +// ".ref..". +// +// For example: 1a3f-2-000001.sst.ref.5.000008 +func (p *provider) sharedObjectRefName(meta objstorage.ObjectMetadata) string { + if meta.Remote.CleanupMethod != objstorage.SharedRefTracking { + panic("ref object used when ref tracking disabled") + } + return sharedObjectRefName(meta, p.remote.shared.creatorID, meta.DiskFileNum) +} + +// objHash returns a 16-bit hash value derived from the creator ID and creator +// file num. We prepend this value to object names to ensure balanced +// partitioning with AWS (and likely other blob storage providers). +func objHash(meta objstorage.ObjectMetadata) uint16 { + const prime1 = 7459 + const prime2 = 17539 + return uint16(uint64(meta.Remote.CreatorID)*prime1 + uint64(meta.Remote.CreatorFileNum.FileNum())*prime2) +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_readable.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_readable.go new file mode 100644 index 0000000..991a1ba --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remote_readable.go @@ -0,0 +1,162 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "context" + "io" + + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache" + "github.com/cockroachdb/pebble/objstorage/remote" +) + +const remoteMaxReadaheadSize = 1024 * 1024 /* 1MB */ + +// remoteReadable is a very simple implementation of Readable on top of the +// ReadCloser returned by remote.Storage.CreateObject. +type remoteReadable struct { + objReader remote.ObjectReader + size int64 + fileNum base.DiskFileNum + provider *provider +} + +var _ objstorage.Readable = (*remoteReadable)(nil) + +func (p *provider) newRemoteReadable( + objReader remote.ObjectReader, size int64, fileNum base.DiskFileNum, +) *remoteReadable { + return &remoteReadable{ + objReader: objReader, + size: size, + fileNum: fileNum, + provider: p, + } +} + +// ReadAt is part of the objstorage.Readable interface. +func (r *remoteReadable) ReadAt(ctx context.Context, p []byte, offset int64) error { + return r.readInternal(ctx, p, offset, false /* forCompaction */) +} + +// readInternal performs a read for the object, using the cache when +// appropriate. +func (r *remoteReadable) readInternal( + ctx context.Context, p []byte, offset int64, forCompaction bool, +) error { + if cache := r.provider.remote.cache; cache != nil { + flags := sharedcache.ReadFlags{ + // Don't add data to the cache if this read is for a compaction. + ReadOnly: forCompaction, + } + return r.provider.remote.cache.ReadAt(ctx, r.fileNum, p, offset, r.objReader, r.size, flags) + } + return r.objReader.ReadAt(ctx, p, offset) +} + +func (r *remoteReadable) Close() error { + defer func() { r.objReader = nil }() + return r.objReader.Close() +} + +func (r *remoteReadable) Size() int64 { + return r.size +} + +func (r *remoteReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle { + // TODO(radu): use a pool. + rh := &remoteReadHandle{readable: r} + rh.readahead.state = makeReadaheadState(remoteMaxReadaheadSize) + return rh +} + +type remoteReadHandle struct { + readable *remoteReadable + readahead struct { + state readaheadState + data []byte + offset int64 + } + forCompaction bool +} + +var _ objstorage.ReadHandle = (*remoteReadHandle)(nil) + +// ReadAt is part of the objstorage.ReadHandle interface. +func (r *remoteReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) error { + readaheadSize := r.maybeReadahead(offset, len(p)) + + // Check if we already have the data from a previous read-ahead. + if rhSize := int64(len(r.readahead.data)); rhSize > 0 { + if r.readahead.offset <= offset && r.readahead.offset+rhSize > offset { + n := copy(p, r.readahead.data[offset-r.readahead.offset:]) + if n == len(p) { + // All data was available. + return nil + } + // Use the data that we had and do a shorter read. + offset += int64(n) + p = p[n:] + readaheadSize -= n + } + } + + if readaheadSize > len(p) { + // Don't try to read past EOF. + if offset+int64(readaheadSize) > r.readable.size { + readaheadSize = int(r.readable.size - offset) + if readaheadSize <= 0 { + // This shouldn't happen in practice (Pebble should never try to read + // past EOF). + return io.EOF + } + } + r.readahead.offset = offset + // TODO(radu): we need to somehow account for this memory. + if cap(r.readahead.data) >= readaheadSize { + r.readahead.data = r.readahead.data[:readaheadSize] + } else { + r.readahead.data = make([]byte, readaheadSize) + } + + if err := r.readable.readInternal(ctx, r.readahead.data, offset, r.forCompaction); err != nil { + // Make sure we don't treat the data as valid next time. + r.readahead.data = r.readahead.data[:0] + return err + } + copy(p, r.readahead.data) + return nil + } + + return r.readable.readInternal(ctx, p, offset, r.forCompaction) +} + +func (r *remoteReadHandle) maybeReadahead(offset int64, len int) int { + if r.forCompaction { + return remoteMaxReadaheadSize + } + return int(r.readahead.state.maybeReadahead(offset, int64(len))) +} + +// Close is part of the objstorage.ReadHandle interface. +func (r *remoteReadHandle) Close() error { + r.readable = nil + r.readahead.data = nil + return nil +} + +// SetupForCompaction is part of the objstorage.ReadHandle interface. +func (r *remoteReadHandle) SetupForCompaction() { + r.forCompaction = true +} + +// RecordCacheHit is part of the objstorage.ReadHandle interface. +func (r *remoteReadHandle) RecordCacheHit(_ context.Context, offset, size int64) { + if !r.forCompaction { + r.readahead.state.recordCacheHit(offset, size) + } +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/sharedobjcat/catalog.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/catalog.go similarity index 73% rename from vendor/github.com/cockroachdb/pebble/objstorage/sharedobjcat/catalog.go rename to vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/catalog.go index 99d9c8b..8508f19 100644 --- a/vendor/github.com/cockroachdb/pebble/objstorage/sharedobjcat/catalog.go +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/catalog.go @@ -2,7 +2,7 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -package sharedobjcat +package remoteobjcat import ( "fmt" @@ -12,23 +12,25 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/remote" "github.com/cockroachdb/pebble/record" "github.com/cockroachdb/pebble/vfs" "github.com/cockroachdb/pebble/vfs/atomicfs" ) -// Catalog is used to manage the on-disk shared object catalog. +// Catalog is used to manage the on-disk remote object catalog. // // The catalog file is a log of records, where each record is an encoded -// versionEdit. +// VersionEdit. type Catalog struct { fs vfs.FS dirname string mu struct { sync.Mutex - creatorID CreatorID - objects map[base.FileNum]SharedObjectMetadata + creatorID objstorage.CreatorID + objects map[base.DiskFileNum]RemoteObjectMetadata marker *atomicfs.Marker @@ -43,44 +45,41 @@ type Catalog struct { } } -// CreatorID identifies the DB instance that originally created a shared object. -// This ID is incorporated in backing object names. -// Must be non-zero. -type CreatorID uint64 - -// IsSet returns true if the CreatorID is not zero. -func (c CreatorID) IsSet() bool { return c != 0 } - -func (c CreatorID) String() string { return fmt.Sprintf("%020d", c) } - -// SharedObjectMetadata encapsulates the data stored in the catalog file for each object. -type SharedObjectMetadata struct { +// RemoteObjectMetadata encapsulates the data stored in the catalog file for each object. +type RemoteObjectMetadata struct { // FileNum is the identifier for the object within the context of a single DB // instance. - FileNum base.FileNum + FileNum base.DiskFileNum // FileType is the type of the object. Only certain FileTypes are possible. FileType base.FileType // CreatorID identifies the DB instance that originally created the object. - CreatorID CreatorID + CreatorID objstorage.CreatorID // CreatorFileNum is the identifier for the object within the context of the // DB instance that originally created the object. - CreatorFileNum base.FileNum + CreatorFileNum base.DiskFileNum + // CleanupMethod indicates the method for cleaning up unused shared objects. + CleanupMethod objstorage.SharedCleanupMethod + // Locator identifies a remote.Storage implementation. + Locator remote.Locator + // CustomObjectName (if it is set) overrides the object name that is normally + // derived from the CreatorID and CreatorFileNum. + CustomObjectName string } const ( - catalogFilenameBase = "SHARED-CATALOG" - catalogMarkerName = "shared-catalog" + catalogFilenameBase = "REMOTE-OBJ-CATALOG" + catalogMarkerName = "remote-obj-catalog" // We create a new file when the size exceeds 1MB (and some other conditions // hold; see record.RotationHelper). rotateFileSize = 1024 * 1024 // 1MB ) -// CatalogContents contains the shared objects in the catalog. +// CatalogContents contains the remote objects in the catalog. type CatalogContents struct { // CreatorID, if it is set. - CreatorID CreatorID - Objects []SharedObjectMetadata + CreatorID objstorage.CreatorID + Objects []RemoteObjectMetadata } // Open creates a Catalog and loads any existing catalog file, returning the @@ -90,7 +89,7 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) { fs: fs, dirname: dirname, } - c.mu.objects = make(map[base.FileNum]SharedObjectMetadata) + c.mu.objects = make(map[base.DiskFileNum]RemoteObjectMetadata) var err error c.mu.marker, c.mu.catalogFilename, err = atomicfs.LocateMarker(fs, dirname, catalogMarkerName) @@ -109,20 +108,20 @@ func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) { } res := CatalogContents{ CreatorID: c.mu.creatorID, - Objects: make([]SharedObjectMetadata, 0, len(c.mu.objects)), + Objects: make([]RemoteObjectMetadata, 0, len(c.mu.objects)), } for _, meta := range c.mu.objects { res.Objects = append(res.Objects, meta) } // Sort the objects so the function is deterministic. sort.Slice(res.Objects, func(i, j int) bool { - return res.Objects[i].FileNum < res.Objects[j].FileNum + return res.Objects[i].FileNum.FileNum() < res.Objects[j].FileNum.FileNum() }) return c, res, nil } // SetCreatorID sets the creator ID. If it is already set, it must match. -func (c *Catalog) SetCreatorID(id CreatorID) error { +func (c *Catalog) SetCreatorID(id objstorage.CreatorID) error { if !id.IsSet() { return errors.AssertionFailedf("attempt to unset CreatorID") } @@ -137,9 +136,9 @@ func (c *Catalog) SetCreatorID(id CreatorID) error { return nil } - ve := versionEdit{CreatorID: id} + ve := VersionEdit{CreatorID: id} if err := c.writeToCatalogFileLocked(&ve); err != nil { - return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err) + return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err) } c.mu.creatorID = id return nil @@ -166,19 +165,19 @@ func (c *Catalog) closeCatalogFile() error { // Batch is used to perform multiple object additions/deletions at once. type Batch struct { - ve versionEdit + ve VersionEdit } // AddObject adds a new object to the batch. // // The given FileNum must be new - it must not match that of any object that was // ever in the catalog. -func (b *Batch) AddObject(meta SharedObjectMetadata) { +func (b *Batch) AddObject(meta RemoteObjectMetadata) { b.ve.NewObjects = append(b.ve.NewObjects, meta) } // DeleteObject adds an object removal to the batch. -func (b *Batch) DeleteObject(fileNum base.FileNum) { +func (b *Batch) DeleteObject(fileNum base.DiskFileNum) { b.ve.DeletedObjects = append(b.ve.DeletedObjects, fileNum) } @@ -197,11 +196,11 @@ func (b *Batch) IsEmpty() bool { func (b *Batch) Copy() Batch { var res Batch if len(b.ve.NewObjects) > 0 { - res.ve.NewObjects = make([]SharedObjectMetadata, len(b.ve.NewObjects)) + res.ve.NewObjects = make([]RemoteObjectMetadata, len(b.ve.NewObjects)) copy(res.ve.NewObjects, b.ve.NewObjects) } if len(b.ve.DeletedObjects) > 0 { - res.ve.DeletedObjects = make([]base.FileNum, len(b.ve.DeletedObjects)) + res.ve.DeletedObjects = make([]base.DiskFileNum, len(b.ve.DeletedObjects)) copy(res.ve.DeletedObjects, b.ve.DeletedObjects) } return res @@ -219,29 +218,40 @@ func (c *Catalog) ApplyBatch(b Batch) error { c.mu.Lock() defer c.mu.Unlock() - for _, n := range b.ve.DeletedObjects { - if _, exists := c.mu.objects[n]; !exists { - return errors.AssertionFailedf("deleting non-existent object %s", n) + // Sanity checks. + toAdd := make(map[base.DiskFileNum]struct{}, len(b.ve.NewObjects)) + exists := func(n base.DiskFileNum) bool { + _, ok := c.mu.objects[n] + if !ok { + _, ok = toAdd[n] } + return ok } for _, meta := range b.ve.NewObjects { - if _, exists := c.mu.objects[meta.FileNum]; exists { + if exists(meta.FileNum) { return errors.AssertionFailedf("adding existing object %s", meta.FileNum) } + toAdd[meta.FileNum] = struct{}{} + } + for _, n := range b.ve.DeletedObjects { + if !exists(n) { + return errors.AssertionFailedf("deleting non-existent object %s", n) + } } if err := c.writeToCatalogFileLocked(&b.ve); err != nil { - return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err) + return errors.Wrapf(err, "pebble: could not write to remote object catalog: %v", err) } - // Apply the batch to our current state. - for _, n := range b.ve.DeletedObjects { - delete(c.mu.objects, n) - } + // Add new objects before deleting any objects. This allows for cases where + // the same batch adds and deletes an object. for _, meta := range b.ve.NewObjects { c.mu.objects[meta.FileNum] = meta } - b.Reset() + for _, n := range b.ve.DeletedObjects { + delete(c.mu.objects, n) + } + return nil } @@ -250,7 +260,7 @@ func (c *Catalog) loadFromCatalogFile(filename string) error { f, err := c.fs.Open(catalogPath) if err != nil { return errors.Wrapf( - err, "pebble: could not open shared object catalog file %q for DB %q", + err, "pebble: could not open remote object catalog file %q for DB %q", errors.Safe(filename), c.dirname, ) } @@ -262,32 +272,26 @@ func (c *Catalog) loadFromCatalogFile(filename string) error { break } if err != nil { - return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q", + return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q", errors.Safe(filename)) } - var ve versionEdit - err = ve.Decode(r) - if err != nil { - return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q", + var ve VersionEdit + if err := ve.Decode(r); err != nil { + return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q", errors.Safe(filename)) } // Apply the version edit to the current state. - if ve.CreatorID.IsSet() { - c.mu.creatorID = ve.CreatorID - } - for _, fileNum := range ve.DeletedObjects { - delete(c.mu.objects, fileNum) - } - for _, meta := range ve.NewObjects { - c.mu.objects[meta.FileNum] = meta + if err := ve.Apply(&c.mu.creatorID, c.mu.objects); err != nil { + return errors.Wrapf(err, "pebble: error when loading remote object catalog file %q", + errors.Safe(filename)) } } return nil } -// writeToCatalogFileLocked writes a versionEdit to the catalog file. +// writeToCatalogFileLocked writes a VersionEdit to the catalog file. // Creates a new file if this is the first write. -func (c *Catalog) writeToCatalogFileLocked(ve *versionEdit) error { +func (c *Catalog) writeToCatalogFileLocked(ve *VersionEdit) error { c.mu.rotationHelper.AddRecord(int64(len(ve.NewObjects) + len(ve.DeletedObjects))) snapshotSize := int64(len(c.mu.objects)) @@ -330,9 +334,10 @@ func (c *Catalog) createNewCatalogFileLocked() (outErr error) { } recWriter := record.NewWriter(file) err = func() error { - // Create a versionEdit that gets us from an empty catalog to the current state. - var ve versionEdit - ve.NewObjects = make([]SharedObjectMetadata, 0, len(c.mu.objects)) + // Create a VersionEdit that gets us from an empty catalog to the current state. + var ve VersionEdit + ve.CreatorID = c.mu.creatorID + ve.NewObjects = make([]RemoteObjectMetadata, 0, len(c.mu.objects)) for _, meta := range c.mu.objects { ve.NewObjects = append(ve.NewObjects, meta) } @@ -367,7 +372,7 @@ func (c *Catalog) createNewCatalogFileLocked() (outErr error) { return nil } -func writeRecord(ve *versionEdit, file vfs.File, recWriter *record.Writer) error { +func writeRecord(ve *VersionEdit, file vfs.File, recWriter *record.Writer) error { w, err := recWriter.Next() if err != nil { return err diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/version_edit.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/version_edit.go new file mode 100644 index 0000000..44552f5 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat/version_edit.go @@ -0,0 +1,254 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package remoteobjcat + +import ( + "bufio" + "encoding/binary" + "io" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/remote" +) + +// VersionEdit is a modification to the remote object state which can be encoded +// into a record. +// +// TODO(radu): consider adding creation and deletion time for debugging purposes. +type VersionEdit struct { + NewObjects []RemoteObjectMetadata + DeletedObjects []base.DiskFileNum + CreatorID objstorage.CreatorID +} + +const ( + // tagNewObject is followed by the FileNum, creator ID, creator FileNum, + // cleanup method, optional new object tags, and ending with a 0 byte. + tagNewObject = 1 + // tagDeletedObject is followed by the FileNum. + tagDeletedObject = 2 + // tagCreatorID is followed by the Creator ID for this store. This ID can + // never change. + tagCreatorID = 3 + // tagNewObjectLocator is an optional tag inside the tagNewObject payload. It + // is followed by the encoded length of the locator string and the string. + tagNewObjectLocator = 4 + // tagNewObjectCustomName is an optional tag inside the tagNewObject payload. + // It is followed by the encoded length of the custom object name string + // followed by the string. + tagNewObjectCustomName = 5 +) + +// Object type values. We don't want to encode FileType directly because it is +// more general (and we want freedom to change it in the future). +const ( + objTypeTable = 1 +) + +func objTypeToFileType(objType uint64) (base.FileType, error) { + switch objType { + case objTypeTable: + return base.FileTypeTable, nil + default: + return 0, errors.Newf("unknown object type %d", objType) + } +} + +func fileTypeToObjType(fileType base.FileType) (uint64, error) { + switch fileType { + case base.FileTypeTable: + return objTypeTable, nil + + default: + return 0, errors.Newf("unknown object type for file type %d", fileType) + } +} + +// Encode encodes an edit to the specified writer. +func (v *VersionEdit) Encode(w io.Writer) error { + buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*10+len(v.DeletedObjects)*2+2)) + for _, meta := range v.NewObjects { + objType, err := fileTypeToObjType(meta.FileType) + if err != nil { + return err + } + buf = binary.AppendUvarint(buf, uint64(tagNewObject)) + buf = binary.AppendUvarint(buf, uint64(meta.FileNum.FileNum())) + buf = binary.AppendUvarint(buf, objType) + buf = binary.AppendUvarint(buf, uint64(meta.CreatorID)) + buf = binary.AppendUvarint(buf, uint64(meta.CreatorFileNum.FileNum())) + buf = binary.AppendUvarint(buf, uint64(meta.CleanupMethod)) + if meta.Locator != "" { + buf = binary.AppendUvarint(buf, uint64(tagNewObjectLocator)) + buf = encodeString(buf, string(meta.Locator)) + } + if meta.CustomObjectName != "" { + buf = binary.AppendUvarint(buf, uint64(tagNewObjectCustomName)) + buf = encodeString(buf, meta.CustomObjectName) + } + // Append 0 as the terminator for optional new object tags. + buf = binary.AppendUvarint(buf, 0) + } + + for _, dfn := range v.DeletedObjects { + buf = binary.AppendUvarint(buf, uint64(tagDeletedObject)) + buf = binary.AppendUvarint(buf, uint64(dfn.FileNum())) + } + if v.CreatorID.IsSet() { + buf = binary.AppendUvarint(buf, uint64(tagCreatorID)) + buf = binary.AppendUvarint(buf, uint64(v.CreatorID)) + } + _, err := w.Write(buf) + return err +} + +// Decode decodes an edit from the specified reader. +func (v *VersionEdit) Decode(r io.Reader) error { + br, ok := r.(io.ByteReader) + if !ok { + br = bufio.NewReader(r) + } + for { + tag, err := binary.ReadUvarint(br) + if err == io.EOF { + break + } + if err != nil { + return err + } + + err = nil + switch tag { + case tagNewObject: + var fileNum, creatorID, creatorFileNum, cleanupMethod uint64 + var locator, customName string + var fileType base.FileType + fileNum, err = binary.ReadUvarint(br) + if err == nil { + var objType uint64 + objType, err = binary.ReadUvarint(br) + if err == nil { + fileType, err = objTypeToFileType(objType) + } + } + if err == nil { + creatorID, err = binary.ReadUvarint(br) + } + if err == nil { + creatorFileNum, err = binary.ReadUvarint(br) + } + if err == nil { + cleanupMethod, err = binary.ReadUvarint(br) + } + for err == nil { + var optionalTag uint64 + optionalTag, err = binary.ReadUvarint(br) + if err != nil || optionalTag == 0 { + break + } + + switch optionalTag { + case tagNewObjectLocator: + locator, err = decodeString(br) + + case tagNewObjectCustomName: + customName, err = decodeString(br) + + default: + err = errors.Newf("unknown newObject tag %d", optionalTag) + } + } + + if err == nil { + v.NewObjects = append(v.NewObjects, RemoteObjectMetadata{ + FileNum: base.FileNum(fileNum).DiskFileNum(), + FileType: fileType, + CreatorID: objstorage.CreatorID(creatorID), + CreatorFileNum: base.FileNum(creatorFileNum).DiskFileNum(), + CleanupMethod: objstorage.SharedCleanupMethod(cleanupMethod), + Locator: remote.Locator(locator), + CustomObjectName: customName, + }) + } + + case tagDeletedObject: + var fileNum uint64 + fileNum, err = binary.ReadUvarint(br) + if err == nil { + v.DeletedObjects = append(v.DeletedObjects, base.FileNum(fileNum).DiskFileNum()) + } + + case tagCreatorID: + var id uint64 + id, err = binary.ReadUvarint(br) + if err == nil { + v.CreatorID = objstorage.CreatorID(id) + } + + default: + err = errors.Newf("unknown tag %d", tag) + } + + if err != nil { + if err == io.EOF { + return errCorruptCatalog + } + return err + } + } + return nil +} + +func encodeString(buf []byte, s string) []byte { + buf = binary.AppendUvarint(buf, uint64(len(s))) + buf = append(buf, []byte(s)...) + return buf +} + +func decodeString(br io.ByteReader) (string, error) { + length, err := binary.ReadUvarint(br) + if err != nil || length == 0 { + return "", err + } + buf := make([]byte, length) + for i := range buf { + buf[i], err = br.ReadByte() + if err != nil { + return "", err + } + } + return string(buf), nil +} + +var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt remote object catalog") + +// Apply the version edit to a creator ID and a map of objects. +func (v *VersionEdit) Apply( + creatorID *objstorage.CreatorID, objects map[base.DiskFileNum]RemoteObjectMetadata, +) error { + if v.CreatorID.IsSet() { + *creatorID = v.CreatorID + } + for _, meta := range v.NewObjects { + if invariants.Enabled { + if _, exists := objects[meta.FileNum]; exists { + return errors.AssertionFailedf("version edit adds existing object %s", meta.FileNum) + } + } + objects[meta.FileNum] = meta + } + for _, fileNum := range v.DeletedObjects { + if invariants.Enabled { + if _, exists := objects[fileNum]; !exists { + return errors.AssertionFailedf("version edit deletes non-existent object %s", fileNum) + } + } + delete(objects, fileNum) + } + return nil +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_writable.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_writable.go new file mode 100644 index 0000000..5e8d45a --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_writable.go @@ -0,0 +1,65 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "io" + + "github.com/cockroachdb/pebble/objstorage" +) + +// NewRemoteWritable creates an objstorage.Writable out of an io.WriteCloser. +func NewRemoteWritable(obj io.WriteCloser) objstorage.Writable { + return &sharedWritable{storageWriter: obj} +} + +// sharedWritable is a very simple implementation of Writable on top of the +// WriteCloser returned by remote.Storage.CreateObject. +type sharedWritable struct { + // Either both p and meta must be unset / zero values, or both must be set. + // The case where both are unset is true only in tests. + p *provider + meta objstorage.ObjectMetadata + storageWriter io.WriteCloser +} + +var _ objstorage.Writable = (*sharedWritable)(nil) + +// Write is part of the Writable interface. +func (w *sharedWritable) Write(p []byte) error { + _, err := w.storageWriter.Write(p) + return err +} + +// Finish is part of the Writable interface. +func (w *sharedWritable) Finish() error { + err := w.storageWriter.Close() + w.storageWriter = nil + if err != nil { + w.Abort() + return err + } + + // Create the marker object. + if w.p != nil { + if err := w.p.sharedCreateRef(w.meta); err != nil { + w.Abort() + return err + } + } + return nil +} + +// Abort is part of the Writable interface. +func (w *sharedWritable) Abort() { + if w.storageWriter != nil { + _ = w.storageWriter.Close() + w.storageWriter = nil + } + if w.p != nil { + w.p.removeMetadata(w.meta.DiskFileNum) + } + // TODO(radu): delete the object if it was created. +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache/shared_cache.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache/shared_cache.go new file mode 100644 index 0000000..6d6409e --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache/shared_cache.go @@ -0,0 +1,900 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package sharedcache + +import ( + "context" + "fmt" + "io" + "math/bits" + "sync" + "sync/atomic" + "time" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/objstorage/remote" + "github.com/cockroachdb/pebble/vfs" + "github.com/prometheus/client_golang/prometheus" +) + +// Exported to enable exporting from package pebble to enable +// exporting metrics with below buckets in CRDB. +var ( + IOBuckets = prometheus.ExponentialBucketsRange(float64(time.Millisecond*1), float64(10*time.Second), 50) + ChannelWriteBuckets = prometheus.ExponentialBucketsRange(float64(time.Microsecond*1), float64(10*time.Second), 50) +) + +// Cache is a persistent cache backed by a local filesystem. It is intended +// to cache data that is in slower shared storage (e.g. S3), hence the +// package name 'sharedcache'. +type Cache struct { + shards []shard + writeWorkers writeWorkers + + bm blockMath + shardingBlockSize int64 + + logger base.Logger + metrics internalMetrics +} + +// Metrics is a struct containing metrics exported by the secondary cache. +// TODO(josh): Reconsider the set of metrics exported by the secondary cache +// before we release the secondary cache to users. We choose to export many metrics +// right now, so we learn a lot from the benchmarking we are doing over the 23.2 +// cycle. +type Metrics struct { + // The number of sstable bytes stored in the cache. + Size int64 + // The count of cache blocks in the cache (not sstable blocks). + Count int64 + + // The number of calls to ReadAt. + TotalReads int64 + // The number of calls to ReadAt that require reading data from 2+ shards. + MultiShardReads int64 + // The number of calls to ReadAt that require reading data from 2+ cache blocks. + MultiBlockReads int64 + // The number of calls to ReadAt where all data returned was read from the cache. + ReadsWithFullHit int64 + // The number of calls to ReadAt where some data returned was read from the cache. + ReadsWithPartialHit int64 + // The number of calls to ReadAt where no data returned was read from the cache. + ReadsWithNoHit int64 + + // The number of times a cache block was evicted from the cache. + Evictions int64 + // The number of times writing a cache block to the cache failed. + WriteBackFailures int64 + + // The latency of calls to get some data from the cache. + GetLatency prometheus.Histogram + // The latency of reads of a single cache block from disk. + DiskReadLatency prometheus.Histogram + // The latency of writing data to write back to the cache to a channel. + // Generally should be low, but if the channel is full, could be high. + QueuePutLatency prometheus.Histogram + // The latency of calls to put some data read from block storage into the cache. + PutLatency prometheus.Histogram + // The latency of writes of a single cache block to disk. + DiskWriteLatency prometheus.Histogram +} + +// See docs at Metrics. +type internalMetrics struct { + count atomic.Int64 + + totalReads atomic.Int64 + multiShardReads atomic.Int64 + multiBlockReads atomic.Int64 + readsWithFullHit atomic.Int64 + readsWithPartialHit atomic.Int64 + readsWithNoHit atomic.Int64 + + evictions atomic.Int64 + writeBackFailures atomic.Int64 + + getLatency prometheus.Histogram + diskReadLatency prometheus.Histogram + queuePutLatency prometheus.Histogram + putLatency prometheus.Histogram + diskWriteLatency prometheus.Histogram +} + +const ( + // writeWorkersPerShard is used to establish the number of worker goroutines + // that perform writes to the cache. + writeWorkersPerShard = 4 + // writeTaskPerWorker is used to establish how many tasks can be queued up + // until we have to block. + writeTasksPerWorker = 4 +) + +// Open opens a cache. If there is no existing cache at fsDir, a new one +// is created. +func Open( + fs vfs.FS, + logger base.Logger, + fsDir string, + blockSize int, + // shardingBlockSize is the size of a shard block. The cache is split into contiguous + // shardingBlockSize units. The units are distributed across multiple independent shards + // of the cache, via a hash(offset) modulo num shards operation. The cache replacement + // policies operate at the level of shard, not whole cache. This is done to reduce lock + // contention. + shardingBlockSize int64, + sizeBytes int64, + numShards int, +) (*Cache, error) { + if minSize := shardingBlockSize * int64(numShards); sizeBytes < minSize { + // Up the size so that we have one block per shard. In practice, this should + // only happen in tests. + sizeBytes = minSize + } + + c := &Cache{ + logger: logger, + bm: makeBlockMath(blockSize), + shardingBlockSize: shardingBlockSize, + } + c.shards = make([]shard, numShards) + blocksPerShard := sizeBytes / int64(numShards) / int64(blockSize) + for i := range c.shards { + if err := c.shards[i].init(c, fs, fsDir, i, blocksPerShard, blockSize, shardingBlockSize); err != nil { + return nil, err + } + } + + c.writeWorkers.Start(c, numShards*writeWorkersPerShard) + + c.metrics.getLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets}) + c.metrics.diskReadLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets}) + c.metrics.putLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets}) + c.metrics.diskWriteLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: IOBuckets}) + + // Measures a channel write, so lower min. + c.metrics.queuePutLatency = prometheus.NewHistogram(prometheus.HistogramOpts{Buckets: ChannelWriteBuckets}) + + return c, nil +} + +// Close closes the cache. Methods such as ReadAt should not be called after Close is +// called. +func (c *Cache) Close() error { + c.writeWorkers.Stop() + + var retErr error + for i := range c.shards { + if err := c.shards[i].close(); err != nil && retErr == nil { + retErr = err + } + } + c.shards = nil + return retErr +} + +// Metrics return metrics for the cache. Callers should not mutate +// the returned histograms, which are pointer types. +func (c *Cache) Metrics() Metrics { + return Metrics{ + Count: c.metrics.count.Load(), + Size: c.metrics.count.Load() * int64(c.bm.BlockSize()), + TotalReads: c.metrics.totalReads.Load(), + MultiShardReads: c.metrics.multiShardReads.Load(), + MultiBlockReads: c.metrics.multiBlockReads.Load(), + ReadsWithFullHit: c.metrics.readsWithFullHit.Load(), + ReadsWithPartialHit: c.metrics.readsWithPartialHit.Load(), + ReadsWithNoHit: c.metrics.readsWithNoHit.Load(), + Evictions: c.metrics.evictions.Load(), + WriteBackFailures: c.metrics.writeBackFailures.Load(), + GetLatency: c.metrics.getLatency, + DiskReadLatency: c.metrics.diskReadLatency, + QueuePutLatency: c.metrics.queuePutLatency, + PutLatency: c.metrics.putLatency, + DiskWriteLatency: c.metrics.diskWriteLatency, + } +} + +// ReadFlags contains options for Cache.ReadAt. +type ReadFlags struct { + // ReadOnly instructs ReadAt to not write any new data into the cache; it is + // used when the data is unlikely to be used again. + ReadOnly bool +} + +// ReadAt performs a read form an object, attempting to use cached data when +// possible. +func (c *Cache) ReadAt( + ctx context.Context, + fileNum base.DiskFileNum, + p []byte, + ofs int64, + objReader remote.ObjectReader, + objSize int64, + flags ReadFlags, +) error { + c.metrics.totalReads.Add(1) + if ofs >= objSize { + if invariants.Enabled { + panic(fmt.Sprintf("invalid ReadAt offset %v %v", ofs, objSize)) + } + return io.EOF + } + // TODO(radu): for compaction reads, we may not want to read from the cache at + // all. + { + start := time.Now() + n, err := c.get(fileNum, p, ofs) + c.metrics.getLatency.Observe(float64(time.Since(start))) + if err != nil { + return err + } + if n == len(p) { + // Everything was in cache! + c.metrics.readsWithFullHit.Add(1) + return nil + } + if n == 0 { + c.metrics.readsWithNoHit.Add(1) + } else { + c.metrics.readsWithPartialHit.Add(1) + } + + // Note this. The below code does not need the original ofs, as with the earlier + // reading from the cache done, the relevant offset is ofs + int64(n). Same with p. + ofs += int64(n) + p = p[n:] + + if invariants.Enabled { + if n != 0 && c.bm.Remainder(ofs) != 0 { + panic(fmt.Sprintf("after non-zero read from cache, ofs is not block-aligned: %v %v", ofs, n)) + } + } + } + + if flags.ReadOnly { + return objReader.ReadAt(ctx, p, ofs) + } + + // We must do reads with offset & size that are multiples of the block size. Else + // later cache hits may return incorrect zeroed results from the cache. + firstBlockInd := c.bm.Block(ofs) + adjustedOfs := c.bm.BlockOffset(firstBlockInd) + + // Take the length of what is left to read plus the length of the adjustment of + // the offset plus the size of a block minus one and divide by the size of a block + // to get the number of blocks to read from the object. + sizeOfOffAdjustment := int(ofs - adjustedOfs) + adjustedLen := int(c.bm.RoundUp(int64(len(p) + sizeOfOffAdjustment))) + adjustedP := make([]byte, adjustedLen) + + // Read the rest from the object. We may need to cap the length to avoid past EOF reads. + eofCap := int64(adjustedLen) + if adjustedOfs+eofCap > objSize { + eofCap = objSize - adjustedOfs + } + if err := objReader.ReadAt(ctx, adjustedP[:eofCap], adjustedOfs); err != nil { + return err + } + copy(p, adjustedP[sizeOfOffAdjustment:]) + + start := time.Now() + c.writeWorkers.QueueWrite(fileNum, adjustedP, adjustedOfs) + c.metrics.queuePutLatency.Observe(float64(time.Since(start))) + + return nil +} + +// get attempts to read the requested data from the cache, if it is already +// there. +// +// If all data is available, returns n = len(p). +// +// If data is partially available, a prefix of the data is read; returns n < len(p) +// and no error. If no prefix is available, returns n = 0 and no error. +func (c *Cache) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ error) { + // The data extent might cross shard boundaries, hence the loop. In the hot + // path, max two iterations of this loop will be executed, since reads are sized + // in units of sstable block size. + var multiShard bool + for { + shard := c.getShard(fileNum, ofs+int64(n)) + cappedLen := len(p[n:]) + if toBoundary := int(c.shardingBlockSize - ((ofs + int64(n)) % c.shardingBlockSize)); cappedLen > toBoundary { + cappedLen = toBoundary + } + numRead, err := shard.get(fileNum, p[n:n+cappedLen], ofs+int64(n)) + if err != nil { + return n, err + } + n += numRead + if numRead < cappedLen { + // We only read a prefix from this shard. + return n, nil + } + if n == len(p) { + // We are done. + return n, nil + } + // Data extent crosses shard boundary, continue with next shard. + if !multiShard { + c.metrics.multiShardReads.Add(1) + multiShard = true + } + } +} + +// set attempts to write the requested data to the cache. Both ofs & len(p) must +// be multiples of the block size. +// +// If all of p is not written to the shard, set returns a non-nil error. +func (c *Cache) set(fileNum base.DiskFileNum, p []byte, ofs int64) error { + if invariants.Enabled { + if c.bm.Remainder(ofs) != 0 || c.bm.Remainder(int64(len(p))) != 0 { + panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p))) + } + } + + // The data extent might cross shard boundaries, hence the loop. In the hot + // path, max two iterations of this loop will be executed, since reads are sized + // in units of sstable block size. + n := 0 + for { + shard := c.getShard(fileNum, ofs+int64(n)) + cappedLen := len(p[n:]) + if toBoundary := int(c.shardingBlockSize - ((ofs + int64(n)) % c.shardingBlockSize)); cappedLen > toBoundary { + cappedLen = toBoundary + } + err := shard.set(fileNum, p[n:n+cappedLen], ofs+int64(n)) + if err != nil { + return err + } + // set returns an error if cappedLen bytes aren't written to the shard. + n += cappedLen + if n == len(p) { + // We are done. + return nil + } + // Data extent crosses shard boundary, continue with next shard. + } +} + +func (c *Cache) getShard(fileNum base.DiskFileNum, ofs int64) *shard { + const prime64 = 1099511628211 + hash := uint64(fileNum.FileNum())*prime64 + uint64(ofs/c.shardingBlockSize) + // TODO(josh): Instance change ops are often run in production. Such an operation + // updates len(c.shards); see openSharedCache. As a result, the behavior of this + // function changes, and the cache empties out at restart time. We may want a better + // story here eventually. + return &c.shards[hash%uint64(len(c.shards))] +} + +type shard struct { + cache *Cache + file vfs.File + sizeInBlocks int64 + bm blockMath + shardingBlockSize int64 + mu struct { + sync.Mutex + // TODO(josh): None of these datastructures are space-efficient. + // Focusing on correctness to start. + where whereMap + blocks []cacheBlockState + // Head of LRU list (doubly-linked circular). + lruHead cacheBlockIndex + // Head of free list (singly-linked chain). + freeHead cacheBlockIndex + } +} + +type cacheBlockState struct { + lock lockState + logical logicalBlockID + + // next is the next block in the LRU or free list (or invalidBlockIndex if it + // is the last block in the free list). + next cacheBlockIndex + + // prev is the previous block in the LRU list. It is not used when the block + // is in the free list. + prev cacheBlockIndex +} + +// Maps a logical block in an SST to an index of the cache block with the +// file contents (to the "cache block index"). +type whereMap map[logicalBlockID]cacheBlockIndex + +type logicalBlockID struct { + filenum base.DiskFileNum + cacheBlockIdx cacheBlockIndex +} + +type lockState int64 + +const ( + unlocked lockState = 0 + // >0 lockState tracks the number of distinct readers of some cache block / logical block + // which is in the secondary cache. It is used to ensure that a cache block is not evicted + // and overwritten, while there are active readers. + readLockTakenInc = 1 + // -1 lockState indicates that some cache block is currently being populated with data from + // blob storage. It is used to ensure that a cache block is not read or evicted again, while + // it is being populated. + writeLockTaken = -1 +) + +func (s *shard) init( + cache *Cache, + fs vfs.FS, + fsDir string, + shardIdx int, + sizeInBlocks int64, + blockSize int, + shardingBlockSize int64, +) error { + *s = shard{ + cache: cache, + sizeInBlocks: sizeInBlocks, + } + if blockSize < 1024 || shardingBlockSize%int64(blockSize) != 0 { + return errors.Newf("invalid block size %d (must divide %d)", blockSize, shardingBlockSize) + } + s.bm = makeBlockMath(blockSize) + s.shardingBlockSize = shardingBlockSize + file, err := fs.OpenReadWrite(fs.PathJoin(fsDir, fmt.Sprintf("SHARED-CACHE-%03d", shardIdx))) + if err != nil { + return err + } + // TODO(radu): truncate file if necessary (especially important if we restart + // with more shards). + if err := file.Preallocate(0, int64(blockSize)*sizeInBlocks); err != nil { + return err + } + s.file = file + + // TODO(josh): Right now, the secondary cache is not persistent. All existing + // cache contents will be over-written, since all metadata is only stored in + // memory. + s.mu.where = make(whereMap) + s.mu.blocks = make([]cacheBlockState, sizeInBlocks) + s.mu.lruHead = invalidBlockIndex + s.mu.freeHead = invalidBlockIndex + for i := range s.mu.blocks { + s.freePush(cacheBlockIndex(i)) + } + + return nil +} + +func (s *shard) close() error { + defer func() { + s.file = nil + }() + return s.file.Close() +} + +// freePush pushes a block to the front of the free list. +func (s *shard) freePush(index cacheBlockIndex) { + s.mu.blocks[index].next = s.mu.freeHead + s.mu.freeHead = index +} + +// freePop removes the block from the front of the free list. Must not be called +// if the list is empty (i.e. freeHead = invalidBlockIndex). +func (s *shard) freePop() cacheBlockIndex { + index := s.mu.freeHead + s.mu.freeHead = s.mu.blocks[index].next + return index +} + +// lruInsertFront inserts a block at the front of the LRU list. +func (s *shard) lruInsertFront(index cacheBlockIndex) { + b := &s.mu.blocks[index] + if s.mu.lruHead == invalidBlockIndex { + b.next = index + b.prev = index + } else { + b.next = s.mu.lruHead + h := &s.mu.blocks[s.mu.lruHead] + b.prev = h.prev + s.mu.blocks[h.prev].next = index + h.prev = index + } + s.mu.lruHead = index +} + +func (s *shard) lruNext(index cacheBlockIndex) cacheBlockIndex { + return s.mu.blocks[index].next +} + +func (s *shard) lruPrev(index cacheBlockIndex) cacheBlockIndex { + return s.mu.blocks[index].prev +} + +// lruUnlink removes a block from the LRU list. +func (s *shard) lruUnlink(index cacheBlockIndex) { + b := &s.mu.blocks[index] + if b.next == index { + s.mu.lruHead = invalidBlockIndex + } else { + s.mu.blocks[b.prev].next = b.next + s.mu.blocks[b.next].prev = b.prev + if s.mu.lruHead == index { + s.mu.lruHead = b.next + } + } + b.next, b.prev = invalidBlockIndex, invalidBlockIndex +} + +// get attempts to read the requested data from the shard. The data must not +// cross a shard boundary. +// +// If all data is available, returns n = len(p). +// +// If data is partially available, a prefix of the data is read; returns n < len(p) +// and no error. If no prefix is available, returns n = 0 and no error. +// +// TODO(josh): Today, if there are two cache blocks needed to satisfy a read, and the +// first block is not in the cache and the second one is, we will read both from +// blob storage. We should fix this. This is not an unlikely scenario if we are doing +// a reverse scan, since those iterate over sstable blocks in reverse order and due to +// cache block aligned reads will have read the suffix of the sstable block that will +// be needed next. +func (s *shard) get(fileNum base.DiskFileNum, p []byte, ofs int64) (n int, _ error) { + if invariants.Enabled { + if ofs/s.shardingBlockSize != (ofs+int64(len(p))-1)/s.shardingBlockSize { + panic(fmt.Sprintf("get crosses shard boundary: %v %v", ofs, len(p))) + } + s.assertShardStateIsConsistent() + } + + // The data extent might cross cache block boundaries, hence the loop. In the hot + // path, max two iterations of this loop will be executed, since reads are sized + // in units of sstable block size. + var multiBlock bool + for { + k := logicalBlockID{ + filenum: fileNum, + cacheBlockIdx: s.bm.Block(ofs + int64(n)), + } + s.mu.Lock() + cacheBlockIdx, ok := s.mu.where[k] + // TODO(josh): Multiple reads within the same few milliseconds (anything that is smaller + // than blob storage read latency) that miss on the same logical block ID will not necessarily + // be rare. We may want to do only one read, with the later readers blocking on the first read + // completing. This could be implemented either here or in the primary block cache. See + // https://github.com/cockroachdb/pebble/pull/2586 for additional discussion. + if !ok { + s.mu.Unlock() + return n, nil + } + if s.mu.blocks[cacheBlockIdx].lock == writeLockTaken { + // In practice, if we have two reads of the same SST block in close succession, we + // would expect the second to hit in the in-memory block cache. So it's not worth + // optimizing this case here. + s.mu.Unlock() + return n, nil + } + s.mu.blocks[cacheBlockIdx].lock += readLockTakenInc + // Move to front of the LRU list. + s.lruUnlink(cacheBlockIdx) + s.lruInsertFront(cacheBlockIdx) + s.mu.Unlock() + + readAt := s.bm.BlockOffset(cacheBlockIdx) + readSize := s.bm.BlockSize() + if n == 0 { // if first read + rem := s.bm.Remainder(ofs) + readAt += rem + readSize -= int(rem) + } + + if len(p[n:]) <= readSize { + start := time.Now() + numRead, err := s.file.ReadAt(p[n:], readAt) + s.cache.metrics.diskReadLatency.Observe(float64(time.Since(start))) + s.dropReadLock(cacheBlockIdx) + return n + numRead, err + } + start := time.Now() + numRead, err := s.file.ReadAt(p[n:n+readSize], readAt) + s.cache.metrics.diskReadLatency.Observe(float64(time.Since(start))) + s.dropReadLock(cacheBlockIdx) + if err != nil { + return 0, err + } + + // Note that numRead == readSize, since we checked for an error above. + n += numRead + + if !multiBlock { + s.cache.metrics.multiBlockReads.Add(1) + multiBlock = true + } + } +} + +// set attempts to write the requested data to the shard. The data must not +// cross a shard boundary, and both ofs & len(p) must be multiples of the +// block size. +// +// If all of p is not written to the shard, set returns a non-nil error. +func (s *shard) set(fileNum base.DiskFileNum, p []byte, ofs int64) error { + if invariants.Enabled { + if ofs/s.shardingBlockSize != (ofs+int64(len(p))-1)/s.shardingBlockSize { + panic(fmt.Sprintf("set crosses shard boundary: %v %v", ofs, len(p))) + } + if s.bm.Remainder(ofs) != 0 || s.bm.Remainder(int64(len(p))) != 0 { + panic(fmt.Sprintf("set with ofs & len not multiples of block size: %v %v", ofs, len(p))) + } + s.assertShardStateIsConsistent() + } + + // The data extent might cross cache block boundaries, hence the loop. In the hot + // path, max two iterations of this loop will be executed, since reads are sized + // in units of sstable block size. + n := 0 + for { + if n == len(p) { + return nil + } + if invariants.Enabled { + if n > len(p) { + panic(fmt.Sprintf("set with n greater than len(p): %v %v", n, len(p))) + } + } + + // If the logical block is already in the cache, we should skip doing a set. + k := logicalBlockID{ + filenum: fileNum, + cacheBlockIdx: s.bm.Block(ofs + int64(n)), + } + s.mu.Lock() + if _, ok := s.mu.where[k]; ok { + s.mu.Unlock() + n += s.bm.BlockSize() + continue + } + + var cacheBlockIdx cacheBlockIndex + if s.mu.freeHead == invalidBlockIndex { + if invariants.Enabled && s.mu.lruHead == invalidBlockIndex { + panic("both LRU and free lists empty") + } + + // Find the last element in the LRU list which is not locked. + for idx := s.lruPrev(s.mu.lruHead); ; idx = s.lruPrev(idx) { + if lock := s.mu.blocks[idx].lock; lock == unlocked { + cacheBlockIdx = idx + break + } + if idx == s.mu.lruHead { + // No unlocked block to evict. + // + // TODO(josh): We may want to block until a block frees up, instead of returning + // an error here. But I think we can do that later on, e.g. after running some production + // experiments. + s.mu.Unlock() + return errors.New("no block to evict so skipping write to cache") + } + } + s.cache.metrics.evictions.Add(1) + s.lruUnlink(cacheBlockIdx) + delete(s.mu.where, s.mu.blocks[cacheBlockIdx].logical) + } else { + s.cache.metrics.count.Add(1) + cacheBlockIdx = s.freePop() + } + + s.lruInsertFront(cacheBlockIdx) + s.mu.where[k] = cacheBlockIdx + s.mu.blocks[cacheBlockIdx].logical = k + s.mu.blocks[cacheBlockIdx].lock = writeLockTaken + s.mu.Unlock() + + writeAt := s.bm.BlockOffset(cacheBlockIdx) + + writeSize := s.bm.BlockSize() + if len(p[n:]) <= writeSize { + writeSize = len(p[n:]) + } + + start := time.Now() + _, err := s.file.WriteAt(p[n:n+writeSize], writeAt) + s.cache.metrics.diskWriteLatency.Observe(float64(time.Since(start))) + if err != nil { + // Free the block. + s.mu.Lock() + defer s.mu.Unlock() + + delete(s.mu.where, k) + s.lruUnlink(cacheBlockIdx) + s.freePush(cacheBlockIdx) + return err + } + s.dropWriteLock(cacheBlockIdx) + n += writeSize + } +} + +// Doesn't inline currently. This might be okay, but something to keep in mind. +func (s *shard) dropReadLock(cacheBlockInd cacheBlockIndex) { + s.mu.Lock() + s.mu.blocks[cacheBlockInd].lock -= readLockTakenInc + if invariants.Enabled && s.mu.blocks[cacheBlockInd].lock < 0 { + panic(fmt.Sprintf("unexpected lock state %v in dropReadLock", s.mu.blocks[cacheBlockInd].lock)) + } + s.mu.Unlock() +} + +// Doesn't inline currently. This might be okay, but something to keep in mind. +func (s *shard) dropWriteLock(cacheBlockInd cacheBlockIndex) { + s.mu.Lock() + if invariants.Enabled && s.mu.blocks[cacheBlockInd].lock != writeLockTaken { + panic(fmt.Sprintf("unexpected lock state %v in dropWriteLock", s.mu.blocks[cacheBlockInd].lock)) + } + s.mu.blocks[cacheBlockInd].lock = unlocked + s.mu.Unlock() +} + +func (s *shard) assertShardStateIsConsistent() { + s.mu.Lock() + defer s.mu.Unlock() + + lruLen := 0 + if s.mu.lruHead != invalidBlockIndex { + for b := s.mu.lruHead; ; { + lruLen++ + if idx, ok := s.mu.where[s.mu.blocks[b].logical]; !ok || idx != b { + panic("block in LRU list with no entry in where map") + } + b = s.lruNext(b) + if b == s.mu.lruHead { + break + } + } + } + if lruLen != len(s.mu.where) { + panic(fmt.Sprintf("lru list len is %d but where map has %d entries", lruLen, len(s.mu.where))) + } + freeLen := 0 + for n := s.mu.freeHead; n != invalidBlockIndex; n = s.mu.blocks[n].next { + freeLen++ + } + + if lruLen+freeLen != int(s.sizeInBlocks) { + panic(fmt.Sprintf("%d lru blocks and %d free blocks don't add up to %d", lruLen, freeLen, s.sizeInBlocks)) + } + for i := range s.mu.blocks { + if state := s.mu.blocks[i].lock; state < writeLockTaken { + panic(fmt.Sprintf("lock state %v is not allowed", state)) + } + } +} + +// cacheBlockIndex is the index of a blockSize-aligned cache block. +type cacheBlockIndex int64 + +// invalidBlockIndex is used for the head of a list when the list is empty. +const invalidBlockIndex cacheBlockIndex = -1 + +// blockMath is a helper type for performing conversions between offsets and +// block indexes. +type blockMath struct { + blockSizeBits int8 +} + +func makeBlockMath(blockSize int) blockMath { + bm := blockMath{ + blockSizeBits: int8(bits.Len64(uint64(blockSize)) - 1), + } + if blockSize != (1 << bm.blockSizeBits) { + panic(fmt.Sprintf("blockSize %d is not a power of 2", blockSize)) + } + return bm +} + +func (bm blockMath) mask() int64 { + return (1 << bm.blockSizeBits) - 1 +} + +// BlockSize returns the block size. +func (bm blockMath) BlockSize() int { + return 1 << bm.blockSizeBits +} + +// Block returns the block index containing the given offset. +func (bm blockMath) Block(offset int64) cacheBlockIndex { + return cacheBlockIndex(offset >> bm.blockSizeBits) +} + +// Remainder returns the offset relative to the start of the cache block. +func (bm blockMath) Remainder(offset int64) int64 { + return offset & bm.mask() +} + +// BlockOffset returns the object offset where the given block starts. +func (bm blockMath) BlockOffset(block cacheBlockIndex) int64 { + return int64(block) << bm.blockSizeBits +} + +// RoundUp rounds up the given value to the closest multiple of block size. +func (bm blockMath) RoundUp(x int64) int64 { + return (x + bm.mask()) & ^(bm.mask()) +} + +type writeWorkers struct { + doneCh chan struct{} + doneWaitGroup sync.WaitGroup + + numWorkers int + tasksCh chan writeTask +} + +type writeTask struct { + fileNum base.DiskFileNum + p []byte + offset int64 +} + +// Start starts the worker goroutines. +func (w *writeWorkers) Start(c *Cache, numWorkers int) { + doneCh := make(chan struct{}) + tasksCh := make(chan writeTask, numWorkers*writeTasksPerWorker) + + w.numWorkers = numWorkers + w.doneCh = doneCh + w.tasksCh = tasksCh + w.doneWaitGroup.Add(numWorkers) + for i := 0; i < numWorkers; i++ { + go func() { + defer w.doneWaitGroup.Done() + for { + select { + case <-doneCh: + return + case task, ok := <-tasksCh: + if !ok { + // The tasks channel was closed; this is used in testing code to + // ensure all writes are completed. + return + } + // TODO(radu): set() can perform multiple writes; perhaps each one + // should be its own task. + start := time.Now() + err := c.set(task.fileNum, task.p, task.offset) + c.metrics.putLatency.Observe(float64(time.Since(start))) + if err != nil { + c.metrics.writeBackFailures.Add(1) + // TODO(radu): throttle logs. + c.logger.Infof("writing back to cache after miss failed: %v", err) + } + } + } + }() + } +} + +// Stop waits for any in-progress writes to complete and stops the worker +// goroutines and waits for any in-pro. Any queued writes not yet started are +// discarded. +func (w *writeWorkers) Stop() { + close(w.doneCh) + w.doneCh = nil + w.tasksCh = nil + w.doneWaitGroup.Wait() +} + +// QueueWrite adds a write task to the queue. Can block if the queue is full. +func (w *writeWorkers) QueueWrite(fileNum base.DiskFileNum, p []byte, offset int64) { + w.tasksCh <- writeTask{ + fileNum: fileNum, + p: p, + offset: offset, + } +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/vfs.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs.go similarity index 60% rename from vendor/github.com/cockroachdb/pebble/objstorage/vfs.go rename to vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs.go index b997a18..5b20251 100644 --- a/vendor/github.com/cockroachdb/pebble/objstorage/vfs.go +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs.go @@ -2,63 +2,63 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -package objstorage +package objstorageprovider import ( + "context" + "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" "github.com/cockroachdb/pebble/vfs" ) -func (p *Provider) vfsPath(fileType base.FileType, fileNum base.FileNum) string { +func (p *provider) vfsPath(fileType base.FileType, fileNum base.DiskFileNum) string { return base.MakeFilepath(p.st.FS, p.st.FSDirName, fileType, fileNum) } -func (p *Provider) vfsOpenForReading( - fileType base.FileType, fileNum base.FileNum, mustExist bool, -) (Readable, error) { +func (p *provider) vfsOpenForReading( + ctx context.Context, + fileType base.FileType, + fileNum base.DiskFileNum, + opts objstorage.OpenOptions, +) (objstorage.Readable, error) { filename := p.vfsPath(fileType, fileNum) file, err := p.st.FS.Open(filename, vfs.RandomReadsOption) if err != nil { - if mustExist { + if opts.MustExist { base.MustExist(p.st.FS, filename, p.st.Logger, err) } return nil, err } - // TODO(radu): we use the existence of the file descriptor as an indication - // that the File might support Prefetch and SequentialReadsOption. We should - // replace this with a cleaner way to obtain the capabilities of the FS / File. - if fd := file.Fd(); fd != vfs.InvalidFd { - return newFileReadable(file, p.st.FS, filename) - } - return newGenericFileReadable(file) + return newFileReadable(file, p.st.FS, filename) } -func (p *Provider) vfsCreate( - fileType base.FileType, fileNum base.FileNum, -) (Writable, ObjectMetadata, error) { +func (p *provider) vfsCreate( + _ context.Context, fileType base.FileType, fileNum base.DiskFileNum, +) (objstorage.Writable, objstorage.ObjectMetadata, error) { filename := p.vfsPath(fileType, fileNum) file, err := p.st.FS.Create(filename) if err != nil { - return nil, ObjectMetadata{}, err + return nil, objstorage.ObjectMetadata{}, err } file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{ NoSyncOnClose: p.st.NoSyncOnClose, BytesPerSync: p.st.BytesPerSync, }) - meta := ObjectMetadata{ - FileNum: fileNum, - FileType: fileType, + meta := objstorage.ObjectMetadata{ + DiskFileNum: fileNum, + FileType: fileType, } return newFileBufferedWritable(file), meta, nil } -func (p *Provider) vfsRemove(fileType base.FileType, fileNum base.FileNum) error { +func (p *provider) vfsRemove(fileType base.FileType, fileNum base.DiskFileNum) error { return p.st.FSCleaner.Clean(p.st.FS, fileType, p.vfsPath(fileType, fileNum)) } // vfsInit finds any local FS objects. -func (p *Provider) vfsInit() error { +func (p *provider) vfsInit() error { listing := p.st.FSDirInitialListing if listing == nil { var err error @@ -71,17 +71,17 @@ func (p *Provider) vfsInit() error { for _, filename := range listing { fileType, fileNum, ok := base.ParseFilename(p.st.FS, filename) if ok && fileType == base.FileTypeTable { - o := ObjectMetadata{ - FileType: fileType, - FileNum: fileNum, + o := objstorage.ObjectMetadata{ + FileType: fileType, + DiskFileNum: fileNum, } - p.mu.knownObjects[o.FileNum] = o + p.mu.knownObjects[o.DiskFileNum] = o } } return nil } -func (p *Provider) vfsSync() error { +func (p *provider) vfsSync() error { p.mu.Lock() shouldSync := p.mu.localObjectsChanged p.mu.localObjectsChanged = false @@ -99,7 +99,7 @@ func (p *Provider) vfsSync() error { return nil } -func (p *Provider) vfsSize(fileType base.FileType, fileNum base.FileNum) (int64, error) { +func (p *provider) vfsSize(fileType base.FileType, fileNum base.DiskFileNum) (int64, error) { filename := p.vfsPath(fileType, fileNum) stat, err := p.st.FS.Stat(filename) if err != nil { diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/vfs_readable.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go similarity index 62% rename from vendor/github.com/cockroachdb/pebble/objstorage/vfs_readable.go rename to vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go index 22749c5..9362464 100644 --- a/vendor/github.com/cockroachdb/pebble/objstorage/vfs_readable.go +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go @@ -2,19 +2,25 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -package objstorage +package objstorageprovider import ( + "context" "fmt" "os" "sync" "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/objstorage" "github.com/cockroachdb/pebble/vfs" ) -// fileReadable implements objstorage.Readable on top of a vfs.File that -// supports Prefetch and/or on an FS that supports SequentialReadsOption. +const fileMaxReadaheadSize = 256 * 1024 /* 256KB */ + +// fileReadable implements objstorage.Readable on top of a vfs.File. +// +// The implementation might use Prealloc and might reopen the file with +// SequentialReadsOption. type fileReadable struct { file vfs.File size int64 @@ -25,7 +31,7 @@ type fileReadable struct { fs vfs.FS } -var _ Readable = (*fileReadable)(nil) +var _ objstorage.Readable = (*fileReadable)(nil) func newFileReadable(file vfs.File, fs vfs.FS, filename string) (*fileReadable, error) { info, err := file.Stat() @@ -48,8 +54,12 @@ func newFileReadable(file vfs.File, fs vfs.FS, filename string) (*fileReadable, } // ReadAt is part of the objstorage.Readable interface. -func (r *fileReadable) ReadAt(p []byte, off int64) (n int, err error) { - return r.file.ReadAt(p, off) +func (r *fileReadable) ReadAt(_ context.Context, p []byte, off int64) error { + n, err := r.file.ReadAt(p, off) + if invariants.Enabled && err == nil && n != len(p) { + panic("short read") + } + return err } // Close is part of the objstorage.Readable interface. @@ -64,10 +74,10 @@ func (r *fileReadable) Size() int64 { } // NewReadHandle is part of the objstorage.Readable interface. -func (r *fileReadable) NewReadHandle() ReadHandle { +func (r *fileReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle { rh := readHandlePool.Get().(*vfsReadHandle) rh.r = r - rh.rs = makeReadaheadState() + rh.rs = makeReadaheadState(fileMaxReadaheadSize) return rh } @@ -82,7 +92,7 @@ type vfsReadHandle struct { sequentialFile vfs.File } -var _ ReadHandle = (*vfsReadHandle)(nil) +var _ objstorage.ReadHandle = (*vfsReadHandle)(nil) var readHandlePool = sync.Pool{ New: func() interface{} { @@ -110,25 +120,36 @@ func (rh *vfsReadHandle) Close() error { } // ReadAt is part of the objstorage.ReadHandle interface. -func (rh *vfsReadHandle) ReadAt(p []byte, offset int64) (n int, err error) { +func (rh *vfsReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error { + var n int + var err error if rh.sequentialFile != nil { // Use OS-level read-ahead. - return rh.sequentialFile.ReadAt(p, offset) - } - if readaheadSize := rh.rs.maybeReadahead(offset, int64(len(p))); readaheadSize > 0 { - if readaheadSize >= maxReadaheadSize { - // We've reached the maximum readahead size. Beyond this point, rely on - // OS-level readahead. - rh.MaxReadahead() - } else { - _ = rh.r.file.Prefetch(offset, readaheadSize) + n, err = rh.sequentialFile.ReadAt(p, offset) + } else { + if readaheadSize := rh.rs.maybeReadahead(offset, int64(len(p))); readaheadSize > 0 { + if readaheadSize >= fileMaxReadaheadSize { + // We've reached the maximum readahead size. Beyond this point, rely on + // OS-level readahead. + rh.switchToOSReadahead() + } else { + _ = rh.r.file.Prefetch(offset, readaheadSize) + } } + n, err = rh.r.file.ReadAt(p, offset) } - return rh.r.file.ReadAt(p, offset) + if invariants.Enabled && err == nil && n != len(p) { + panic("short read") + } + return err } -// MaxReadahead is part of the objstorage.ReadHandle interface. -func (rh *vfsReadHandle) MaxReadahead() { +// SetupForCompaction is part of the objstorage.ReadHandle interface. +func (rh *vfsReadHandle) SetupForCompaction() { + rh.switchToOSReadahead() +} + +func (rh *vfsReadHandle) switchToOSReadahead() { if rh.sequentialFile != nil { return } @@ -142,7 +163,7 @@ func (rh *vfsReadHandle) MaxReadahead() { } // RecordCacheHit is part of the objstorage.ReadHandle interface. -func (rh *vfsReadHandle) RecordCacheHit(offset, size int64) { +func (rh *vfsReadHandle) RecordCacheHit(_ context.Context, offset, size int64) { if rh.sequentialFile != nil { // Using OS-level readahead, so do nothing. return @@ -150,60 +171,9 @@ func (rh *vfsReadHandle) RecordCacheHit(offset, size int64) { rh.rs.recordCacheHit(offset, size) } -// genericFileReadable implements objstorage.Readable on top of any vfs.File. -// This implementation does not support read-ahead. -type genericFileReadable struct { - file vfs.File - size int64 - - rh NoopReadHandle -} - -var _ Readable = (*genericFileReadable)(nil) - -func newGenericFileReadable(file vfs.File) (*genericFileReadable, error) { - info, err := file.Stat() - if err != nil { - return nil, err - } - r := &genericFileReadable{ - file: file, - size: info.Size(), - rh: MakeNoopReadHandle(file), - } - invariants.SetFinalizer(r, func(obj interface{}) { - if obj.(*genericFileReadable).file != nil { - fmt.Fprintf(os.Stderr, "Readable was not closed") - os.Exit(1) - } - }) - return r, nil -} - -// ReadAt is part of the objstorage.Readable interface. -func (r *genericFileReadable) ReadAt(p []byte, off int64) (n int, err error) { - return r.file.ReadAt(p, off) -} - -// Close is part of the objstorage.Readable interface. -func (r *genericFileReadable) Close() error { - defer func() { r.file = nil }() - return r.file.Close() -} - -// Size is part of the objstorage.Readable interface. -func (r *genericFileReadable) Size() int64 { - return r.size -} - -// NewReadHandle is part of the objstorage.Readable interface. -func (r *genericFileReadable) NewReadHandle() ReadHandle { - return &r.rh -} - // TestingCheckMaxReadahead returns true if the ReadHandle has switched to // OS-level read-ahead. -func TestingCheckMaxReadahead(rh ReadHandle) bool { +func TestingCheckMaxReadahead(rh objstorage.ReadHandle) bool { switch rh := rh.(type) { case *vfsReadHandle: return rh.sequentialFile != nil @@ -234,11 +204,13 @@ func (rh *PreallocatedReadHandle) Close() error { // but uses the existing storage of a PreallocatedReadHandle when possible // (currently this happens if we are reading from a local file). // The returned handle still needs to be closed. -func UsePreallocatedReadHandle(readable Readable, rh *PreallocatedReadHandle) ReadHandle { +func UsePreallocatedReadHandle( + ctx context.Context, readable objstorage.Readable, rh *PreallocatedReadHandle, +) objstorage.ReadHandle { if r, ok := readable.(*fileReadable); ok { // See fileReadable.NewReadHandle. rh.vfsReadHandle = vfsReadHandle{r: r} return rh } - return readable.NewReadHandle() + return readable.NewReadHandle(ctx) } diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/vfs_writable.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_writable.go similarity index 87% rename from vendor/github.com/cockroachdb/pebble/objstorage/vfs_writable.go rename to vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_writable.go index 376d42c..c9b207c 100644 --- a/vendor/github.com/cockroachdb/pebble/objstorage/vfs_writable.go +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_writable.go @@ -2,16 +2,17 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -package objstorage +package objstorageprovider import ( "bufio" + "github.com/cockroachdb/pebble/objstorage" "github.com/cockroachdb/pebble/vfs" ) // NewFileWritable returns a Writable that uses a file as underlying storage. -func NewFileWritable(file vfs.File) Writable { +func NewFileWritable(file vfs.File) objstorage.Writable { return newFileBufferedWritable(file) } @@ -20,7 +21,7 @@ type fileBufferedWritable struct { bw *bufio.Writer } -var _ Writable = (*fileBufferedWritable)(nil) +var _ objstorage.Writable = (*fileBufferedWritable)(nil) func newFileBufferedWritable(file vfs.File) *fileBufferedWritable { return &fileBufferedWritable{ diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/provider.go b/vendor/github.com/cockroachdb/pebble/objstorage/provider.go deleted file mode 100644 index 145abe7..0000000 --- a/vendor/github.com/cockroachdb/pebble/objstorage/provider.go +++ /dev/null @@ -1,446 +0,0 @@ -// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -package objstorage - -import ( - "io" - "os" - "sort" - "sync" - - "github.com/cockroachdb/errors" - "github.com/cockroachdb/errors/oserror" - "github.com/cockroachdb/pebble/internal/base" - "github.com/cockroachdb/pebble/objstorage/shared" - "github.com/cockroachdb/pebble/objstorage/sharedobjcat" - "github.com/cockroachdb/pebble/vfs" -) - -// Provider is a singleton object used to access and manage objects. -// -// An object is conceptually like a large immutable file. The main use of -// objects is for storing sstables; in the future it could also be used for blob -// storage. -// -// The Provider can only manage objects that it knows about - either objects -// created by the provider, or existing objects the Provider was informed about -// via AddObjects. -// -// Objects are currently backed by a vfs.File. -type Provider struct { - st Settings - - fsDir vfs.File - - shared sharedSubsystem - - mu struct { - sync.RWMutex - - shared struct { - // catalogBatch accumulates shared object creations and deletions until - // Sync is called. - catalogBatch sharedobjcat.Batch - } - - // localObjectsChanged is set if non-shared objects were created or deleted - // but Sync was not yet called. - localObjectsChanged bool - - // knownObjects maintains information about objects that are known to the provider. - // It is initialized with the list of files in the manifest when we open a DB. - knownObjects map[base.FileNum]ObjectMetadata - } -} - -// Readable is the handle for an object that is open for reading. -type Readable interface { - io.ReaderAt - io.Closer - - // Size returns the size of the object. - Size() int64 - - // NewReadHandle creates a read handle for ReadAt requests that are related - // and can benefit from optimizations like read-ahead. - // - // The ReadHandle must be closed before the Readable is closed. - // - // Multiple separate ReadHandles can be used. - NewReadHandle() ReadHandle -} - -// ReadHandle is used to perform reads that are related and might benefit from -// optimizations like read-ahead. -type ReadHandle interface { - io.ReaderAt - io.Closer - - // MaxReadahead configures the implementation to expect large sequential - // reads. Used to skip any initial read-ahead ramp-up. - MaxReadahead() - - // RecordCacheHit informs the implementation that we were able to retrieve a - // block from cache. - RecordCacheHit(offset, size int64) -} - -// Writable is the handle for an object that is open for writing. -// Either Finish or Abort must be called. -type Writable interface { - // Write writes len(p) bytes from p to the underlying object. The data is not - // guaranteed to be durable until Finish is called. - // - // Note that Write *is* allowed to modify the slice passed in, whether - // temporarily or permanently. Callers of Write need to take this into - // account. - Write(p []byte) error - - // Finish completes the object and makes the data durable. - // No further calls are allowed after calling Finish. - Finish() error - - // Abort gives up on finishing the object. There is no guarantee about whether - // the object exists after calling Abort. - // No further calls are allowed after calling Abort. - Abort() -} - -// Settings that must be specified when creating the Provider. -type Settings struct { - Logger base.Logger - - // Local filesystem configuration. - FS vfs.FS - FSDirName string - - // FSDirInitialListing is a listing of FSDirName at the time of calling Open. - // - // This is an optional optimization to avoid double listing on Open when the - // higher layer already has a listing. When nil, we obtain the listing on - // Open. - FSDirInitialListing []string - - // Cleaner cleans obsolete files from the local filesystem. - // - // The default cleaner uses the DeleteCleaner. - FSCleaner base.Cleaner - - // NoSyncOnClose decides whether the implementation will enforce a - // close-time synchronization (e.g., fdatasync() or sync_file_range()) - // on files it writes to. Setting this to true removes the guarantee for a - // sync on close. Some implementations can still issue a non-blocking sync. - NoSyncOnClose bool - - // BytesPerSync enables periodic syncing of files in order to smooth out - // writes to disk. This option does not provide any persistence guarantee, but - // is used to avoid latency spikes if the OS automatically decides to write - // out a large chunk of dirty filesystem buffers. - BytesPerSync int - - // Fields here are set only if the provider is to support shared objects - // (experimental). - Shared struct { - Storage shared.Storage - } -} - -// DefaultSettings initializes default settings (with no shared storage), -// suitable for tests and tools. -func DefaultSettings(fs vfs.FS, dirName string) Settings { - return Settings{ - Logger: base.DefaultLogger, - FS: fs, - FSDirName: dirName, - FSCleaner: base.DeleteCleaner{}, - NoSyncOnClose: false, - BytesPerSync: 512 * 1024, // 512KB - } -} - -// ObjectMetadata contains the metadata required to be able to access an object. -type ObjectMetadata struct { - FileNum base.FileNum - FileType base.FileType - - // The fields below are only set if the object is on shared storage. - Shared struct { - // CreatorID identifies the DB instance that originally created the object. - CreatorID CreatorID - // CreatorFileNum is the identifier for the object within the context of the - // DB instance that originally created the object. - CreatorFileNum base.FileNum - } -} - -// CreatorID identifies the DB instance that originally created a shared object. -// This ID is incorporated in backing object names. -// Must be non-zero. -type CreatorID = sharedobjcat.CreatorID - -// IsShared returns true if the object is on shared storage. -func (meta *ObjectMetadata) IsShared() bool { - return meta.Shared.CreatorID.IsSet() -} - -// Open creates the Provider. -func Open(settings Settings) (p *Provider, _ error) { - fsDir, err := settings.FS.OpenDir(settings.FSDirName) - if err != nil { - return nil, err - } - - defer func() { - if p == nil { - fsDir.Close() - } - }() - - p = &Provider{ - st: settings, - fsDir: fsDir, - } - p.mu.knownObjects = make(map[base.FileNum]ObjectMetadata) - - // Add local FS objects. - if err := p.vfsInit(); err != nil { - return nil, err - } - - // Initialize shared subsystem (if configured) and add shared objects. - if err := p.sharedInit(); err != nil { - return nil, err - } - - return p, nil -} - -// Close the provider. -func (p *Provider) Close() error { - var err error - if p.fsDir != nil { - err = p.fsDir.Close() - p.fsDir = nil - } - return err -} - -// OpenForReading opens an existing object. -func (p *Provider) OpenForReading(fileType base.FileType, fileNum base.FileNum) (Readable, error) { - meta, err := p.Lookup(fileType, fileNum) - if err != nil { - return nil, err - } - - if !meta.IsShared() { - return p.vfsOpenForReading(fileType, fileNum, false /* mustExist */) - } - return p.sharedOpenForReading(meta) -} - -// OpenForReadingMustExist is a variant of OpenForReading which causes a fatal -// error if the file does not exist. The fatal error message contains -// information helpful for debugging. -func (p *Provider) OpenForReadingMustExist( - fileType base.FileType, fileNum base.FileNum, -) (Readable, error) { - meta, err := p.Lookup(fileType, fileNum) - if err != nil { - p.st.Logger.Fatalf("%v", err) - return nil, err - } - - if !meta.IsShared() { - return p.vfsOpenForReading(fileType, fileNum, true /* mustExist */) - } - - // TODO(radu): implement "must exist" behavior. - return p.sharedOpenForReading(meta) -} - -// CreateOptions contains optional arguments for Create. -type CreateOptions struct { - // PreferSharedStorage causes the object to be created on shared storage if - // the provider has shared storage configured. - PreferSharedStorage bool -} - -// Create creates a new object and opens it for writing. -// -// The object is not guaranteed to be durable (accessible in case of crashes) -// until Sync is called. -func (p *Provider) Create( - fileType base.FileType, fileNum base.FileNum, opts CreateOptions, -) (w Writable, meta ObjectMetadata, err error) { - if opts.PreferSharedStorage && p.st.Shared.Storage != nil { - w, meta, err = p.sharedCreate(fileType, fileNum) - } else { - w, meta, err = p.vfsCreate(fileType, fileNum) - } - if err != nil { - err = errors.Wrapf(err, "creating object %s", errors.Safe(fileNum)) - return nil, ObjectMetadata{}, err - } - p.addMetadata(meta) - return w, meta, nil -} - -// Remove removes an object. -// -// The object is not guaranteed to be durably removed until Sync is called. -func (p *Provider) Remove(fileType base.FileType, fileNum base.FileNum) error { - meta, err := p.Lookup(fileType, fileNum) - if err != nil { - return err - } - - if !meta.IsShared() { - err = p.vfsRemove(fileType, fileNum) - } - // TODO(radu): implement shared object removal (i.e. deref). - - if err != nil && !IsNotExistError(err) { - // We want to be able to retry a Remove, so we keep the object in our list. - // TODO(radu): we should mark the object as "zombie" and not allow any other - // operations. - return errors.Wrapf(err, "removing object %s", errors.Safe(fileNum)) - } - p.removeMetadata(fileNum) - return err -} - -// IsNotExistError indicates whether the error is known to report that a file or -// directory does not exist. -func IsNotExistError(err error) bool { - return oserror.IsNotExist(err) -} - -// Sync flushes the metadata from creation or removal of objects since the last Sync. -func (p *Provider) Sync() error { - if err := p.vfsSync(); err != nil { - return err - } - if err := p.sharedSync(); err != nil { - return err - } - return nil -} - -// LinkOrCopyFromLocal creates a new object that is either a copy of a given -// local file or a hard link (if the new object is created on the same FS, and -// if the FS supports it). -// -// The object is not guaranteed to be durable (accessible in case of crashes) -// until Sync is called. -func (p *Provider) LinkOrCopyFromLocal( - srcFS vfs.FS, srcFilePath string, dstFileType base.FileType, dstFileNum base.FileNum, -) (ObjectMetadata, error) { - if srcFS == p.st.FS { - // Wrap the normal filesystem with one which wraps newly created files with - // vfs.NewSyncingFile. - fs := vfs.NewSyncingFS(p.st.FS, vfs.SyncingFileOptions{ - NoSyncOnClose: p.st.NoSyncOnClose, - BytesPerSync: p.st.BytesPerSync, - }) - dstPath := p.vfsPath(dstFileType, dstFileNum) - if err := vfs.LinkOrCopy(fs, srcFilePath, dstPath); err != nil { - return ObjectMetadata{}, err - } - - meta := ObjectMetadata{ - FileNum: dstFileNum, - FileType: dstFileType, - } - p.addMetadata(meta) - return meta, nil - } - // TODO(radu): for the copy case, we should use `p.Create` and do the copy ourselves. - panic("unimplemented") -} - -// Lookup returns the metadata of an object that is already known to the Provider. -// Does not perform any I/O. -func (p *Provider) Lookup(fileType base.FileType, fileNum base.FileNum) (ObjectMetadata, error) { - p.mu.RLock() - defer p.mu.RUnlock() - meta, ok := p.mu.knownObjects[fileNum] - if !ok { - return ObjectMetadata{}, errors.Wrapf( - os.ErrNotExist, - "file %s (type %d) unknown to the objstorage provider", - errors.Safe(fileNum), errors.Safe(fileType), - ) - } - if meta.FileType != fileType { - return ObjectMetadata{}, errors.AssertionFailedf( - "file %s type mismatch (known type %d, expected type %d)", - errors.Safe(fileNum), errors.Safe(meta.FileType), errors.Safe(fileType), - ) - } - return meta, nil -} - -// Path returns an internal, implementation-dependent path for the object. It is -// meant to be used for informational purposes (like logging). -func (p *Provider) Path(meta ObjectMetadata) string { - if !meta.IsShared() { - return p.vfsPath(meta.FileType, meta.FileNum) - } - return p.sharedPath(meta) -} - -// Size returns the size of the object. -func (p *Provider) Size(meta ObjectMetadata) (int64, error) { - if !meta.IsShared() { - return p.vfsSize(meta.FileType, meta.FileNum) - } - return p.sharedSize(meta) -} - -// List returns the objects currently known to the provider. Does not perform any I/O. -func (p *Provider) List() []ObjectMetadata { - p.mu.RLock() - defer p.mu.RUnlock() - res := make([]ObjectMetadata, 0, len(p.mu.knownObjects)) - for _, meta := range p.mu.knownObjects { - res = append(res, meta) - } - sort.Slice(res, func(i, j int) bool { - return res[i].FileNum < res[j].FileNum - }) - return res -} - -func (p *Provider) addMetadata(meta ObjectMetadata) { - p.mu.Lock() - defer p.mu.Unlock() - p.mu.knownObjects[meta.FileNum] = meta - if meta.IsShared() { - p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{ - FileNum: meta.FileNum, - FileType: meta.FileType, - CreatorID: meta.Shared.CreatorID, - CreatorFileNum: meta.Shared.CreatorFileNum, - }) - } else { - p.mu.localObjectsChanged = true - } -} - -func (p *Provider) removeMetadata(fileNum base.FileNum) { - p.mu.Lock() - defer p.mu.Unlock() - - meta, ok := p.mu.knownObjects[fileNum] - if !ok { - return - } - delete(p.mu.knownObjects, fileNum) - if meta.IsShared() { - p.mu.shared.catalogBatch.DeleteObject(fileNum) - } else { - p.mu.localObjectsChanged = true - } -} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/remote/factory.go b/vendor/github.com/cockroachdb/pebble/objstorage/remote/factory.go new file mode 100644 index 0000000..620dbba --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/remote/factory.go @@ -0,0 +1,25 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package remote + +import "github.com/pkg/errors" + +// MakeSimpleFactory returns a StorageFactory implementation that produces the given +// Storage objects. +func MakeSimpleFactory(m map[Locator]Storage) StorageFactory { + return simpleFactory(m) +} + +type simpleFactory map[Locator]Storage + +var _ StorageFactory = simpleFactory{} + +// CreateStorage is part of the StorageFactory interface. +func (sf simpleFactory) CreateStorage(locator Locator) (Storage, error) { + if s, ok := sf[locator]; ok { + return s, nil + } + return nil, errors.Errorf("unknown locator '%s'", locator) +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/remote/localfs.go b/vendor/github.com/cockroachdb/pebble/objstorage/remote/localfs.go new file mode 100644 index 0000000..539ecb1 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/remote/localfs.go @@ -0,0 +1,118 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package remote + +import ( + "context" + "io" + "os" + "path" + + "github.com/cockroachdb/pebble/vfs" +) + +// NewLocalFS returns a vfs-backed implementation of the remote.Storage +// interface (for testing). All objects will be stored at the directory +// dirname. +func NewLocalFS(dirname string, fs vfs.FS) Storage { + store := &localFSStore{ + dirname: dirname, + vfs: fs, + } + return store +} + +// localFSStore is a vfs-backed implementation of the remote.Storage +// interface (for testing). +type localFSStore struct { + dirname string + vfs vfs.FS +} + +var _ Storage = (*localFSStore)(nil) + +// Close is part of the remote.Storage interface. +func (s *localFSStore) Close() error { + *s = localFSStore{} + return nil +} + +// ReadObject is part of the remote.Storage interface. +func (s *localFSStore) ReadObject( + ctx context.Context, objName string, +) (_ ObjectReader, objSize int64, _ error) { + f, err := s.vfs.Open(path.Join(s.dirname, objName)) + if err != nil { + return nil, 0, err + } + stat, err := f.Stat() + if err != nil { + return nil, 0, err + } + + return &localFSReader{f}, stat.Size(), nil +} + +type localFSReader struct { + file vfs.File +} + +var _ ObjectReader = (*localFSReader)(nil) + +// ReadAt is part of the shared.ObjectReader interface. +func (r *localFSReader) ReadAt(_ context.Context, p []byte, offset int64) error { + n, err := r.file.ReadAt(p, offset) + // https://pkg.go.dev/io#ReaderAt + if err == io.EOF && n == len(p) { + return nil + } + return err +} + +// Close is part of the shared.ObjectReader interface. +func (r *localFSReader) Close() error { + r.file.Close() + r.file = nil + return nil +} + +// CreateObject is part of the remote.Storage interface. +func (s *localFSStore) CreateObject(objName string) (io.WriteCloser, error) { + file, err := s.vfs.Create(path.Join(s.dirname, objName)) + return file, err +} + +// List is part of the remote.Storage interface. +func (s *localFSStore) List(prefix, delimiter string) ([]string, error) { + // TODO(josh): For the intended use case of localfs.go of running 'pebble bench', + // List can always return , since this indicates a file has only one ref, + // and since `pebble bench` implies running in a single-pebble-instance context. + // https://github.com/cockroachdb/pebble/blob/a9a079d4fb6bf4a9ebc52e4d83a76ad4cbf676cb/objstorage/objstorageprovider/shared.go#L292 + return nil, nil +} + +// Delete is part of the remote.Storage interface. +func (s *localFSStore) Delete(objName string) error { + return s.vfs.Remove(path.Join(s.dirname, objName)) +} + +// Size is part of the remote.Storage interface. +func (s *localFSStore) Size(objName string) (int64, error) { + f, err := s.vfs.Open(path.Join(s.dirname, objName)) + if err != nil { + return 0, err + } + defer f.Close() + stat, err := f.Stat() + if err != nil { + return 0, err + } + return stat.Size(), nil +} + +// IsNotExistError is part of the remote.Storage interface. +func (s *localFSStore) IsNotExistError(err error) bool { + return err == os.ErrNotExist +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/shared/logging.go b/vendor/github.com/cockroachdb/pebble/objstorage/remote/logging.go similarity index 57% rename from vendor/github.com/cockroachdb/pebble/objstorage/shared/logging.go rename to vendor/github.com/cockroachdb/pebble/objstorage/remote/logging.go index 5449d11..7a9cd77 100644 --- a/vendor/github.com/cockroachdb/pebble/objstorage/shared/logging.go +++ b/vendor/github.com/cockroachdb/pebble/objstorage/remote/logging.go @@ -2,9 +2,10 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -package shared +package remote import ( + "context" "fmt" "io" "sort" @@ -19,7 +20,7 @@ func WithLogging(wrapped Storage, logf func(fmt string, args ...interface{})) St } } -// loggingStore wraps a shared.Storage implementation and emits logs of the +// loggingStore wraps a remote.Storage implementation and emits logs of the // operations. type loggingStore struct { logf func(fmt string, args ...interface{}) @@ -33,41 +34,52 @@ func (l *loggingStore) Close() error { return l.wrapped.Close() } -func (l *loggingStore) ReadObjectAt( - basename string, offset int64, -) (_ io.ReadCloser, totalSize int64, _ error) { - r, totalSize, err := l.wrapped.ReadObjectAt(basename, offset) - l.logf("read object %q at %d: %s", basename, offset, errOrPrintf(err, "%d bytes", totalSize)) +func (l *loggingStore) ReadObject( + ctx context.Context, objName string, +) (_ ObjectReader, objSize int64, _ error) { + r, size, err := l.wrapped.ReadObject(ctx, objName) + l.logf("create reader for object %q: %s", objName, errOrPrintf(err, "%d bytes", size)) if err != nil { return nil, 0, err } return &loggingReader{ - l: l, - name: basename, - ReadCloser: r, - }, totalSize, nil + l: l, + name: objName, + wrapped: r, + }, size, nil } type loggingReader struct { - l *loggingStore - name string - io.ReadCloser + l *loggingStore + name string + wrapped ObjectReader +} + +var _ ObjectReader = (*loggingReader)(nil) + +func (l *loggingReader) ReadAt(ctx context.Context, p []byte, offset int64) error { + if err := l.wrapped.ReadAt(ctx, p, offset); err != nil { + l.l.logf("read object %q at %d (length %d): error %v", l.name, offset, len(p), err) + return err + } + l.l.logf("read object %q at %d (length %d)", l.name, offset, len(p)) + return nil } func (l *loggingReader) Close() error { l.l.logf("close reader for %q", l.name) - return l.ReadCloser.Close() + return l.wrapped.Close() } -func (l *loggingStore) CreateObject(basename string) (io.WriteCloser, error) { - l.logf("create object %q", basename) - writer, err := l.wrapped.CreateObject(basename) +func (l *loggingStore) CreateObject(objName string) (io.WriteCloser, error) { + l.logf("create object %q", objName) + writer, err := l.wrapped.CreateObject(objName) if err != nil { return nil, err } return &loggingWriter{ l: l, - name: basename, + name: objName, WriteCloser: writer, }, nil } @@ -104,14 +116,14 @@ func (l *loggingStore) List(prefix, delimiter string) ([]string, error) { return res, nil } -func (l *loggingStore) Delete(basename string) error { - l.logf("delete object %q", basename) - return l.wrapped.Delete(basename) +func (l *loggingStore) Delete(objName string) error { + l.logf("delete object %q", objName) + return l.wrapped.Delete(objName) } -func (l *loggingStore) Size(basename string) (int64, error) { - size, err := l.wrapped.Size(basename) - l.logf("size of object %q: %s", basename, errOrPrintf(err, "%d", size)) +func (l *loggingStore) Size(objName string) (int64, error) { + size, err := l.wrapped.Size(objName) + l.logf("size of object %q: %s", objName, errOrPrintf(err, "%d", size)) return size, err } @@ -121,3 +133,7 @@ func errOrPrintf(err error, format string, args ...interface{}) string { } return fmt.Sprintf(format, args...) } + +func (l *loggingStore) IsNotExistError(err error) bool { + return l.wrapped.IsNotExistError(err) +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/shared/mem.go b/vendor/github.com/cockroachdb/pebble/objstorage/remote/mem.go similarity index 69% rename from vendor/github.com/cockroachdb/pebble/objstorage/shared/mem.go rename to vendor/github.com/cockroachdb/pebble/objstorage/remote/mem.go index 72cc7c0..9bbda71 100644 --- a/vendor/github.com/cockroachdb/pebble/objstorage/shared/mem.go +++ b/vendor/github.com/cockroachdb/pebble/objstorage/remote/mem.go @@ -2,17 +2,18 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -package shared +package remote import ( "bytes" + "context" "io" "os" "strings" "sync" ) -// NewInMem returns an in-memory implementation of the shared.Storage +// NewInMem returns an in-memory implementation of the remote.Storage // interface (for testing). func NewInMem() Storage { store := &inMemStore{} @@ -20,7 +21,7 @@ func NewInMem() Storage { return store } -// inMemStore is an in-memory implementation of the shared.Storage interface +// inMemStore is an in-memory implementation of the remote.Storage interface // (for testing). type inMemStore struct { mu struct { @@ -41,39 +42,39 @@ func (s *inMemStore) Close() error { return nil } -func (s *inMemStore) ReadObjectAt(basename string, offset int64) (io.ReadCloser, int64, error) { - obj, err := s.getObj(basename) +func (s *inMemStore) ReadObject( + ctx context.Context, objName string, +) (_ ObjectReader, objSize int64, _ error) { + obj, err := s.getObj(objName) if err != nil { return nil, 0, err } - remaining := int64(len(obj.data)) - offset - if remaining < 0 { - return nil, 0, io.EOF - } - return newInMemReader(obj.data[int(offset):]), remaining, nil + return &inMemReader{data: obj.data}, int64(len(obj.data)), nil } type inMemReader struct { - bytes.Reader + data []byte } -func newInMemReader(b []byte) *inMemReader { - r := &inMemReader{} - r.Reset(b) - return r -} +var _ ObjectReader = (*inMemReader)(nil) -var _ io.ReadCloser = (*inMemReader)(nil) - -func (r *inMemReader) Close() error { - r.Reset(nil) +func (r *inMemReader) ReadAt(ctx context.Context, p []byte, offset int64) error { + if offset+int64(len(p)) > int64(len(r.data)) { + return io.EOF + } + copy(p, r.data[offset:]) return nil } -func (s *inMemStore) CreateObject(basename string) (io.WriteCloser, error) { +func (r *inMemReader) Close() error { + r.data = nil + return nil +} + +func (s *inMemStore) CreateObject(objName string) (io.WriteCloser, error) { return &inMemWriter{ store: s, - name: basename, + name: objName, }, nil } @@ -119,20 +120,24 @@ func (s *inMemStore) List(prefix, delimiter string) ([]string, error) { return res, nil } -func (s *inMemStore) Delete(basename string) error { - s.rmObj(basename) +func (s *inMemStore) Delete(objName string) error { + s.rmObj(objName) return nil } // Size returns the length of the named object in bytesWritten. -func (s *inMemStore) Size(basename string) (int64, error) { - obj, err := s.getObj(basename) +func (s *inMemStore) Size(objName string) (int64, error) { + obj, err := s.getObj(objName) if err != nil { return 0, err } return int64(len(obj.data)), nil } +func (s *inMemStore) IsNotExistError(err error) bool { + return err == os.ErrNotExist +} + func (s *inMemStore) getObj(name string) (*inMemObj, error) { s.mu.Lock() defer s.mu.Unlock() diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/remote/storage.go b/vendor/github.com/cockroachdb/pebble/objstorage/remote/storage.go new file mode 100644 index 0000000..dccb006 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/remote/storage.go @@ -0,0 +1,126 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package remote + +import ( + "context" + "io" +) + +// Locator is an opaque string identifying a remote.Storage implementation. +// +// The Locator must not contain secrets (like authentication keys). Locators are +// stored on disk in the shared object catalog and are passed around as part of +// RemoteObjectBacking; they can also appear in error messages. +type Locator string + +// StorageFactory is used to return Storage implementations based on locators. A +// Pebble store that uses shared storage is configured with a StorageFactory. +type StorageFactory interface { + CreateStorage(locator Locator) (Storage, error) +} + +// SharedLevelsStart denotes the highest (i.e. lowest numbered) level that will +// have sstables shared across Pebble instances when doing skip-shared +// iteration (see db.ScanInternal) or shared file ingestion (see +// db.IngestAndExcise). +const SharedLevelsStart = 5 + +// CreateOnSharedStrategy specifies what table files should be created on shared +// storage. For use with CreateOnShared in options. +type CreateOnSharedStrategy int + +const ( + // CreateOnSharedNone denotes no files being created on shared storage. + CreateOnSharedNone CreateOnSharedStrategy = iota + // CreateOnSharedLower denotes the creation of files in lower levels of the + // LSM (specifically, L5 and L6 as they're below SharedLevelsStart) on + // shared storage, and higher levels on local storage. + CreateOnSharedLower + // CreateOnSharedAll denotes the creation of all sstables on shared storage. + CreateOnSharedAll +) + +// ShouldCreateShared returns whether new table files at the specified level +// should be created on shared storage. +func ShouldCreateShared(strategy CreateOnSharedStrategy, level int) bool { + switch strategy { + case CreateOnSharedAll: + return true + case CreateOnSharedNone: + return false + case CreateOnSharedLower: + return level >= SharedLevelsStart + default: + panic("unexpected CreateOnSharedStrategy value") + } +} + +// Storage is an interface for a blob storage driver. This is lower-level +// than an FS-like interface, however FS/File-like abstractions can be built on +// top of these methods. +// +// TODO(bilal): Consider pushing shared file obsoletion as well as path +// generation behind this interface. +type Storage interface { + io.Closer + + // ReadObject returns an ObjectReader that can be used to perform reads on an + // object, along with the total size of the object. + ReadObject(ctx context.Context, objName string) (_ ObjectReader, objSize int64, _ error) + + // CreateObject returns a writer for the object at the request name. A new + // empty object is created if CreateObject is called on an existing object. + // + // A Writer *must* be closed via either Close, and if closing returns a + // non-nil error, that error should be handled or reported to the user -- an + // implementation may buffer written data until Close and only then return + // an error, or Write may return an opaque io.EOF with the underlying cause + // returned by the subsequent Close(). + // + // TODO(radu): if we encounter some unrelated error while writing to the + // WriteCloser, we'd want to abort the whole thing rather than letting Close + // finalize the upload. + CreateObject(objName string) (io.WriteCloser, error) + + // List enumerates files within the supplied prefix, returning a list of + // objects within that prefix. If delimiter is non-empty, names which have the + // same prefix, prior to the delimiter but after the prefix, are grouped into a + // single result which is that prefix. The order that results are returned is + // undefined. If a prefix is specified, the prefix is trimmed from the result + // list. + // + // An example would be, if the storage contains objects a, b/4, b/5 and b/6, + // these would be the return values: + // List("", "") -> ["a", "b/4", "b/5", "b/6"] + // List("", "/") -> ["a", "b"] + // List("b", "/") -> ["4", "5", "6"] + // List("b", "") -> ["/4", "/5", "/6"] + List(prefix, delimiter string) ([]string, error) + + // Delete removes the named object from the store. + Delete(objName string) error + + // Size returns the length of the named object in bytesWritten. + Size(objName string) (int64, error) + + // IsNotExistError returns true if the given error (returned by a method in + // this interface) indicates that the object does not exist. + IsNotExistError(err error) bool +} + +// ObjectReader is used to perform reads on an object. +type ObjectReader interface { + // ReadAt reads len(p) bytes into p starting at offset off. + // + // Does not return partial results; if offset + len(p) is past the end of the + // object, an error is returned. + // + // Clients of ReadAt can execute parallel ReadAt calls on the same + // ObjectReader. + ReadAt(ctx context.Context, p []byte, offset int64) error + + Close() error +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/shared.go b/vendor/github.com/cockroachdb/pebble/objstorage/shared.go deleted file mode 100644 index 5d9652e..0000000 --- a/vendor/github.com/cockroachdb/pebble/objstorage/shared.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -package objstorage - -import ( - "fmt" - "sync" - "sync/atomic" - - "github.com/cockroachdb/errors" - "github.com/cockroachdb/pebble/internal/base" - "github.com/cockroachdb/pebble/objstorage/sharedobjcat" -) - -// sharedSubsystem contains the provider fields related to shared storage. -// All fields remain unset if shared storage is not configured. -type sharedSubsystem struct { - catalog *sharedobjcat.Catalog - // initialized guards access to the creatorID field. - initialized atomic.Bool - creatorID CreatorID - initOnce sync.Once -} - -func (ss *sharedSubsystem) init(creatorID CreatorID) { - ss.initOnce.Do(func() { - ss.creatorID = creatorID - ss.initialized.Store(true) - }) -} - -// sharedInit initializes the shared object subsystem (if configured) and finds -// any shared objects. -func (p *Provider) sharedInit() error { - if p.st.Shared.Storage == nil { - return nil - } - catalog, contents, err := sharedobjcat.Open(p.st.FS, p.st.FSDirName) - if err != nil { - return errors.Wrapf(err, "pebble: could not open shared object catalog") - } - p.shared.catalog = catalog - - // The creator ID may or may not be initialized yet. - if contents.CreatorID.IsSet() { - p.shared.init(contents.CreatorID) - p.st.Logger.Infof("shared storage configured; creatorID = %s", contents.CreatorID) - } else { - p.st.Logger.Infof("shared storage configured; no creatorID yet") - } - - for _, meta := range contents.Objects { - o := ObjectMetadata{ - FileNum: meta.FileNum, - FileType: meta.FileType, - } - o.Shared.CreatorID = meta.CreatorID - o.Shared.CreatorFileNum = meta.CreatorFileNum - p.mu.knownObjects[o.FileNum] = o - } - return nil -} - -// SetCreatorID sets the CreatorID which is needed in order to use shared -// objects. Shared object usage is disabled until this method is called the -// first time. Once set, the Creator ID is persisted and cannot change. -// -// Cannot be called if shared storage is not configured for the provider. -func (p *Provider) SetCreatorID(creatorID CreatorID) error { - if p.st.Shared.Storage == nil { - return errors.AssertionFailedf("attempt to set CreatorID but shared storage not enabled") - } - // Note: this call is a cheap no-op if the creator ID was already set. This - // call also checks if we are trying to change the ID. - if err := p.shared.catalog.SetCreatorID(creatorID); err != nil { - return err - } - if !p.shared.initialized.Load() { - p.st.Logger.Infof("shared storage creatorID set to %s", creatorID) - p.shared.init(creatorID) - } - return nil -} - -func (p *Provider) sharedCheckInitialized() error { - if p.st.Shared.Storage == nil { - return errors.Errorf("shared object support not configured") - } - if !p.shared.initialized.Load() { - return errors.Errorf("shared object support not available: shared creator ID not yet set") - } - return nil -} - -func (p *Provider) sharedSync() error { - batch := func() sharedobjcat.Batch { - p.mu.Lock() - defer p.mu.Unlock() - res := p.mu.shared.catalogBatch.Copy() - p.mu.shared.catalogBatch.Reset() - return res - }() - - if batch.IsEmpty() { - return nil - } - - err := p.shared.catalog.ApplyBatch(batch) - if err != nil { - // We have to put back the batch (for the next Sync). - p.mu.Lock() - defer p.mu.Unlock() - batch.Append(p.mu.shared.catalogBatch) - p.mu.shared.catalogBatch = batch - return err - } - - return nil -} - -func (p *Provider) sharedPath(meta ObjectMetadata) string { - return "shared://" + sharedObjectName(meta) -} - -func sharedObjectName(meta ObjectMetadata) string { - // TODO(radu): prepend a "shard" value for better distribution within the bucket? - return fmt.Sprintf( - "%s-%s", - meta.Shared.CreatorID, base.MakeFilename(meta.FileType, meta.Shared.CreatorFileNum), - ) -} - -func (p *Provider) sharedCreate( - fileType base.FileType, fileNum base.FileNum, -) (Writable, ObjectMetadata, error) { - if err := p.sharedCheckInitialized(); err != nil { - return nil, ObjectMetadata{}, err - } - meta := ObjectMetadata{ - FileNum: fileNum, - FileType: fileType, - } - meta.Shared.CreatorID = p.shared.creatorID - meta.Shared.CreatorFileNum = fileNum - - objName := sharedObjectName(meta) - writer, err := p.st.Shared.Storage.CreateObject(objName) - if err != nil { - return nil, ObjectMetadata{}, err - } - return &sharedWritable{ - storageWriter: writer, - }, meta, nil -} - -func (p *Provider) sharedOpenForReading(meta ObjectMetadata) (Readable, error) { - if err := p.sharedCheckInitialized(); err != nil { - return nil, err - } - objName := sharedObjectName(meta) - size, err := p.st.Shared.Storage.Size(objName) - if err != nil { - return nil, err - } - return newSharedReadable(p.st.Shared.Storage, objName, size), nil -} - -func (p *Provider) sharedSize(meta ObjectMetadata) (int64, error) { - if err := p.sharedCheckInitialized(); err != nil { - return 0, err - } - objName := sharedObjectName(meta) - return p.st.Shared.Storage.Size(objName) -} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/shared/storage.go b/vendor/github.com/cockroachdb/pebble/objstorage/shared/storage.go deleted file mode 100644 index ae8a516..0000000 --- a/vendor/github.com/cockroachdb/pebble/objstorage/shared/storage.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -package shared - -import "io" - -// Storage is an interface for a blob storage driver. This is lower-level -// than an FS-like interface, however FS/File-like abstractions can be built on -// top of these methods. -// -// TODO(bilal): Consider pushing shared file obsoletion as well as path -// generation behind this interface. -type Storage interface { - io.Closer - - // ReadObjectAt returns a Reader for reading the object at the requested name - // and offset, along with the total size of the object. - ReadObjectAt(basename string, offset int64) (_ io.ReadCloser, totalSize int64, _ error) - - // CreateObject returns a writer for the object at the request name. A new - // empty object is created if CreateObject is called on an existing object. - // - // A Writer *must* be closed via either Close, and if closing returns a - // non-nil error, that error should be handled or reported to the user -- an - // implementation may buffer written data until Close and only then return - // an error, or Write may return an opaque io.EOF with the underlying cause - // returned by the subsequent Close(). - // - // TODO(radu): if we encounter some unrelated error while writing to the - // WriteCloser, we'd want to abort the whole thing rather than letting Close - // finalize the upload. - CreateObject(basename string) (io.WriteCloser, error) - - // List enumerates files within the supplied prefix, returning a list of - // objects within that prefix. If delimiter is non-empty, names which have the - // same prefix, prior to the delimiter but after the prefix, are grouped into a - // single result which is that prefix. The order that results are returned is - // undefined. If a prefix is specified, the prefix is trimmed from the result - // list. - // - // An example would be, if the storage contains objects a, b/4, b/5 and b/6, - // these would be the return values: - // List("", "") -> ["a", "b/4", "b/5", "b/6"] - // List("", "/") -> ["a", "b"] - // List("b", "/") -> ["4", "5", "6"] - // List("b", "") -> ["/4", "/5", "/6"] - List(prefix, delimiter string) ([]string, error) - - // Delete removes the named object from the store. - // - // TODO(radu): do we get an error when the object isn't there? How can we - // check for that error? - Delete(basename string) error - - // Size returns the length of the named object in bytesWritten. - // - // TODO(radu): same as above - how can we tell if it's a "no such object" error? - Size(basename string) (int64, error) -} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/shared_backing.go b/vendor/github.com/cockroachdb/pebble/objstorage/shared_backing.go deleted file mode 100644 index cacf898..0000000 --- a/vendor/github.com/cockroachdb/pebble/objstorage/shared_backing.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -package objstorage - -import ( - "bytes" - "encoding/binary" - "io" - - "github.com/cockroachdb/errors" - "github.com/cockroachdb/pebble/internal/base" - "github.com/cockroachdb/pebble/objstorage/sharedobjcat" -) - -// SharedObjectBacking encodes the metadata necessary to incorporate a shared -// object into a different Pebble instance. -type SharedObjectBacking []byte - -const ( - tagCreatorID = 1 - tagCreatorFileNum = 2 - - // Any new tags that don't have the tagNotSafeToIgnoreMask bit set must be - // followed by the length of the data (so they can be skipped). - - // Any new tags that have the tagNotSafeToIgnoreMask bit set cause errors if - // they are encountered by earlier code that doesn't know the tag. - tagNotSafeToIgnoreMask = 64 -) - -// SharedObjectBacking encodes the shared object metadata. -func (meta *ObjectMetadata) SharedObjectBacking() (SharedObjectBacking, error) { - if !meta.IsShared() { - return nil, errors.AssertionFailedf("object %s not on shared storage", meta.FileNum) - } - - buf := make([]byte, 0, binary.MaxVarintLen64*4) - buf = binary.AppendUvarint(buf, tagCreatorID) - buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorID)) - buf = binary.AppendUvarint(buf, tagCreatorFileNum) - buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorFileNum)) - return buf, nil -} - -// fromSharedObjectBacking decodes the shared object metadata. -func fromSharedObjectBacking( - fileType base.FileType, fileNum base.FileNum, buf SharedObjectBacking, -) (ObjectMetadata, error) { - var creatorID uint64 - var creatorFileNum uint64 - br := bytes.NewReader(buf) - for { - tag, err := binary.ReadUvarint(br) - if err == io.EOF { - break - } - if err != nil { - return ObjectMetadata{}, err - } - switch tag { - case tagCreatorID: - creatorID, err = binary.ReadUvarint(br) - - case tagCreatorFileNum: - creatorFileNum, err = binary.ReadUvarint(br) - - // TODO(radu): encode file type as well? - - default: - // Ignore unknown tags, unless they're not safe to ignore. - if tag&tagNotSafeToIgnoreMask != 0 { - return ObjectMetadata{}, errors.Newf("unknown tag %d", tag) - } - var dataLen uint64 - dataLen, err = binary.ReadUvarint(br) - if err == nil { - _, err = br.Seek(int64(dataLen), io.SeekCurrent) - } - } - if err != nil { - return ObjectMetadata{}, err - } - } - if creatorID == 0 { - return ObjectMetadata{}, errors.Newf("shared object backing missing creator ID") - } - if creatorFileNum == 0 { - return ObjectMetadata{}, errors.Newf("shared object backing missing creator file num") - } - meta := ObjectMetadata{ - FileNum: fileNum, - FileType: fileType, - } - meta.Shared.CreatorID = CreatorID(creatorID) - meta.Shared.CreatorFileNum = base.FileNum(creatorFileNum) - return meta, nil -} - -// SharedObjectToAttach contains the arguments needed to attach an existing shared object. -type SharedObjectToAttach struct { - // FileNum is the file number that will be used to refer to this object (in - // the context of this instance). - FileNum base.FileNum - FileType base.FileType - // Backing contains the metadata for the share dobject backing (normally - // generated from a different instance). - Backing SharedObjectBacking -} - -// AttachSharedObjects registers existing shared objects with this provider. -func (p *Provider) AttachSharedObjects(objs []SharedObjectToAttach) ([]ObjectMetadata, error) { - metas := make([]ObjectMetadata, len(objs)) - for i, o := range objs { - meta, err := fromSharedObjectBacking(o.FileType, o.FileNum, o.Backing) - if err != nil { - return nil, err - } - metas[i] = meta - } - - func() { - p.mu.Lock() - defer p.mu.Unlock() - for _, meta := range metas { - p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{ - FileNum: meta.FileNum, - FileType: meta.FileType, - CreatorID: meta.Shared.CreatorID, - CreatorFileNum: meta.Shared.CreatorFileNum, - }) - } - }() - if err := p.sharedSync(); err != nil { - return nil, err - } - - p.mu.Lock() - defer p.mu.Unlock() - for _, meta := range metas { - p.mu.knownObjects[meta.FileNum] = meta - } - return metas, nil -} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/shared_readable.go b/vendor/github.com/cockroachdb/pebble/objstorage/shared_readable.go deleted file mode 100644 index 5bd32f7..0000000 --- a/vendor/github.com/cockroachdb/pebble/objstorage/shared_readable.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -package objstorage - -import ( - "io" - - "github.com/cockroachdb/pebble/objstorage/shared" -) - -// sharedReadable is a very simple implementation of Readable on top of the -// ReadCloser returned by shared.Storage.CreateObject. -type sharedReadable struct { - storage shared.Storage - objName string - size int64 - - // rh is used for direct ReadAt calls without a read handle. - rh sharedReadHandle -} - -var _ Readable = (*sharedReadable)(nil) - -func newSharedReadable(storage shared.Storage, objName string, size int64) *sharedReadable { - r := &sharedReadable{ - storage: storage, - objName: objName, - size: size, - } - r.rh.readable = r - return r -} - -func (r *sharedReadable) ReadAt(p []byte, offset int64) (n int, err error) { - return r.rh.ReadAt(p, offset) -} - -func (r *sharedReadable) Close() error { - err := r.rh.Close() - r.storage = nil - return err -} - -func (r *sharedReadable) Size() int64 { - return r.size -} - -func (r *sharedReadable) NewReadHandle() ReadHandle { - // TODO(radu): use a pool. - return &sharedReadHandle{readable: r} -} - -type sharedReadHandle struct { - readable *sharedReadable - lastReader io.ReadCloser - lastOffset int64 -} - -var _ ReadHandle = (*sharedReadHandle)(nil) - -func (r *sharedReadHandle) ReadAt(p []byte, offset int64) (n int, err error) { - // See if this continues the previous read so that we can reuse the last reader. - if r.lastReader == nil || r.lastOffset != offset { - // We need to create a new reader. - if r.lastReader != nil { - if err := r.lastReader.Close(); err != nil { - return 0, err - } - r.lastReader = nil - } - reader, _, err := r.readable.storage.ReadObjectAt(r.readable.objName, offset) - if err != nil { - return 0, err - } - r.lastReader = reader - r.lastOffset = offset - } - n, err = io.ReadFull(r.lastReader, p) - r.lastOffset += int64(n) - return n, err -} - -func (r *sharedReadHandle) Close() error { - var err error - if r.lastReader != nil { - err = r.lastReader.Close() - r.lastReader = nil - } - r.readable = nil - return err -} - -func (r *sharedReadHandle) MaxReadahead() {} - -func (r *sharedReadHandle) RecordCacheHit(offset, size int64) {} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/shared_writable.go b/vendor/github.com/cockroachdb/pebble/objstorage/shared_writable.go deleted file mode 100644 index 2c5e444..0000000 --- a/vendor/github.com/cockroachdb/pebble/objstorage/shared_writable.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -package objstorage - -import "io" - -// sharedWritable is a very simple implementation of Writable on top of the -// WriteCloser returned by shared.Storage.CreateObject. -type sharedWritable struct { - storageWriter io.WriteCloser -} - -var _ Writable = (*sharedWritable)(nil) - -// Write is part of the Writable interface. -func (w *sharedWritable) Write(p []byte) error { - _, err := w.storageWriter.Write(p) - return err -} - -// Finish is part of the Writable interface. -func (w *sharedWritable) Finish() error { - err := w.storageWriter.Close() - w.storageWriter = nil - return err -} - -// Abort is part of the Writable interface. -func (w *sharedWritable) Abort() { - _ = w.storageWriter.Close() - w.storageWriter = nil -} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/sharedobjcat/version_edit.go b/vendor/github.com/cockroachdb/pebble/objstorage/sharedobjcat/version_edit.go deleted file mode 100644 index e6da72b..0000000 --- a/vendor/github.com/cockroachdb/pebble/objstorage/sharedobjcat/version_edit.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -package sharedobjcat - -import ( - "bufio" - "encoding/binary" - "io" - - "github.com/cockroachdb/errors" - "github.com/cockroachdb/pebble/internal/base" -) - -// versionEdit is a modification to the shared object state which can be encoded -// into a record. -// -// TODO(radu): consider adding creation and deletion time for debugging purposes. -type versionEdit struct { - NewObjects []SharedObjectMetadata - DeletedObjects []base.FileNum - CreatorID CreatorID -} - -const ( - // tagNewObject is followed by the FileNum, creator ID and creator FileNum. - tagNewObject = 1 - // tagDeletedObject is followed by the FileNum. - tagDeletedObject = 2 - // tagCreatorID is followed by the Creator ID for this store. This ID can - // never change. - tagCreatorID = 3 -) - -// Object type values. We don't want to encode FileType directly because it is -// more general (and we want freedom to change it in the future). -const ( - objTypeTable = 1 -) - -func objTypeToFileType(objType uint64) (base.FileType, error) { - switch objType { - case objTypeTable: - return base.FileTypeTable, nil - default: - return 0, errors.Newf("unknown object type %d", objType) - } -} - -func fileTypeToObjType(fileType base.FileType) (uint64, error) { - switch fileType { - case base.FileTypeTable: - return objTypeTable, nil - - default: - return 0, errors.Newf("unknown object type for file type %d", fileType) - } -} - -// Encode encodes an edit to the specified writer. -func (v *versionEdit) Encode(w io.Writer) error { - buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*4+len(v.DeletedObjects)*2+2)) - for _, meta := range v.NewObjects { - objType, err := fileTypeToObjType(meta.FileType) - if err != nil { - return err - } - buf = binary.AppendUvarint(buf, uint64(tagNewObject)) - buf = binary.AppendUvarint(buf, uint64(meta.FileNum)) - buf = binary.AppendUvarint(buf, objType) - buf = binary.AppendUvarint(buf, uint64(meta.CreatorID)) - buf = binary.AppendUvarint(buf, uint64(meta.CreatorFileNum)) - } - - for _, fileNum := range v.DeletedObjects { - buf = binary.AppendUvarint(buf, uint64(tagDeletedObject)) - buf = binary.AppendUvarint(buf, uint64(fileNum)) - } - if v.CreatorID.IsSet() { - buf = binary.AppendUvarint(buf, uint64(tagCreatorID)) - buf = binary.AppendUvarint(buf, uint64(v.CreatorID)) - } - _, err := w.Write(buf) - return err -} - -// Decode decodes an edit from the specified reader. -func (v *versionEdit) Decode(r io.Reader) error { - br, ok := r.(io.ByteReader) - if !ok { - br = bufio.NewReader(r) - } - for { - tag, err := binary.ReadUvarint(br) - if err == io.EOF { - break - } - if err != nil { - return err - } - - err = nil - switch tag { - case tagNewObject: - var fileNum, creatorID, creatorFileNum uint64 - var fileType base.FileType - fileNum, err = binary.ReadUvarint(br) - if err == nil { - var objType uint64 - objType, err = binary.ReadUvarint(br) - if err == nil { - fileType, err = objTypeToFileType(objType) - } - } - if err == nil { - creatorID, err = binary.ReadUvarint(br) - } - if err == nil { - creatorFileNum, err = binary.ReadUvarint(br) - } - if err == nil { - v.NewObjects = append(v.NewObjects, SharedObjectMetadata{ - FileNum: base.FileNum(fileNum), - FileType: fileType, - CreatorID: CreatorID(creatorID), - CreatorFileNum: base.FileNum(creatorFileNum), - }) - } - - case tagDeletedObject: - var fileNum uint64 - fileNum, err = binary.ReadUvarint(br) - if err == nil { - v.DeletedObjects = append(v.DeletedObjects, base.FileNum(fileNum)) - } - - case tagCreatorID: - var id uint64 - id, err = binary.ReadUvarint(br) - if err == nil { - v.CreatorID = CreatorID(id) - } - - default: - // Ignore unknown tags. - } - if err != nil { - if err == io.EOF { - return errCorruptCatalog - } - return err - } - } - return nil -} - -var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt shared object catalog") diff --git a/vendor/github.com/cockroachdb/pebble/open.go b/vendor/github.com/cockroachdb/pebble/open.go index 519b285..89b7c96 100644 --- a/vendor/github.com/cockroachdb/pebble/open.go +++ b/vendor/github.com/cockroachdb/pebble/open.go @@ -6,6 +6,7 @@ package pebble import ( "bytes" + "context" "encoding/binary" "fmt" "io" @@ -20,11 +21,14 @@ import ( "github.com/cockroachdb/pebble/internal/arenaskl" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/cache" + "github.com/cockroachdb/pebble/internal/constants" "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/internal/manual" "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider" "github.com/cockroachdb/pebble/record" + "github.com/cockroachdb/pebble/sstable" "github.com/cockroachdb/pebble/vfs" "github.com/prometheus/client_golang/prometheus" ) @@ -34,11 +38,23 @@ const ( // The max batch size is limited by the uint32 offsets stored in // internal/batchskl.node, DeferredBatchOp, and flushableBatchEntry. - maxBatchSize = 4 << 30 // 4 GB + // + // We limit the size to MaxUint32 (just short of 4GB) so that the exclusive + // end of an allocation fits in uint32. + // + // On 32-bit systems, slices are naturally limited to MaxInt (just short of + // 2GB). + maxBatchSize = constants.MaxUint32OrInt // The max memtable size is limited by the uint32 offsets stored in // internal/arenaskl.node, DeferredBatchOp, and flushableBatchEntry. - maxMemTableSize = 4 << 30 // 4 GB + // + // We limit the size to MaxUint32 (just short of 4GB) so that the exclusive + // end of an allocation fits in uint32. + // + // On 32-bit systems, slices are naturally limited to MaxInt (just short of + // 2GB). + maxMemTableSize = constants.MaxUint32OrInt ) // TableCacheSize can be used to determine the table @@ -157,7 +173,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { merge: opts.Merger.Merge, split: opts.Comparer.Split, abbreviatedKey: opts.Comparer.AbbreviatedKey, - largeBatchThreshold: (opts.MemTableSize - int(memTableEmptySize)) / 2, + largeBatchThreshold: (opts.MemTableSize - uint64(memTableEmptySize)) / 2, fileLock: fileLock, dataDir: dataDir, walDir: walDir, @@ -166,8 +182,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { closedCh: make(chan struct{}), } d.mu.versions = &versionSet{} - d.atomic.diskAvailBytes = math.MaxUint64 - d.mu.versions.diskAvailBytes = d.getDiskAvailableBytesCached + d.diskAvailBytes.Store(math.MaxUint64) defer func() { // If an error or panic occurs during open, attempt to release the manually @@ -192,6 +207,9 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { t.arenaBuf = nil } } + if d.cleanupManager != nil { + d.cleanupManager.Close() + } if d.objProvider != nil { d.objProvider.Close() } @@ -202,34 +220,30 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { }() d.commit = newCommitPipeline(commitEnv{ - logSeqNum: &d.mu.versions.atomic.logSeqNum, - visibleSeqNum: &d.mu.versions.atomic.visibleSeqNum, + logSeqNum: &d.mu.versions.logSeqNum, + visibleSeqNum: &d.mu.versions.visibleSeqNum, apply: d.commitApply, write: d.commitWrite, }) - if d.opts.Experimental.MinDeletionRate > 0 { - d.deletionPacer = newDeletionPacer(int64(d.opts.Experimental.MinDeletionRate), d.getDeletionPacerInfo) - } else { - d.deletionPacer = nilPacer - } d.mu.nextJobID = 1 d.mu.mem.nextSize = opts.MemTableSize if d.mu.mem.nextSize > initialMemTableSize { d.mu.mem.nextSize = initialMemTableSize } - d.mu.mem.cond.L = &d.mu.Mutex - d.mu.cleaner.cond.L = &d.mu.Mutex d.mu.compact.cond.L = &d.mu.Mutex d.mu.compact.inProgress = make(map[*compaction]struct{}) d.mu.compact.noOngoingFlushStartTime = time.Now() d.mu.snapshots.init() - // logSeqNum is the next sequence number that will be assigned. Start - // assigning sequence numbers from 1 to match rocksdb. - d.mu.versions.atomic.logSeqNum = 1 + // logSeqNum is the next sequence number that will be assigned. + // Start assigning sequence numbers from base.SeqNumStart to leave + // room for reserved sequence numbers (see comments around + // SeqNumStart). + d.mu.versions.logSeqNum.Store(base.SeqNumStart) d.mu.formatVers.vers.Store(uint64(formatVersion)) d.mu.formatVers.marker = formatVersionMarker d.timeNow = time.Now + d.openedAt = d.timeNow() d.mu.Lock() defer d.mu.Unlock() @@ -246,7 +260,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { } // Create the DB. - if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, setCurrent, &d.mu.Mutex); err != nil { + if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, setCurrent, d.FormatMajorVersion, &d.mu.Mutex); err != nil { return nil, err } } else { @@ -254,11 +268,11 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { return nil, errors.Wrapf(ErrDBAlreadyExists, "dirname=%q", dirname) } // Load the version set. - if err := d.mu.versions.load(dirname, opts, manifestFileNum, manifestMarker, setCurrent, &d.mu.Mutex); err != nil { + if err := d.mu.versions.load(dirname, opts, manifestFileNum.FileNum(), manifestMarker, setCurrent, d.FormatMajorVersion, &d.mu.Mutex); err != nil { return nil, err } if opts.ErrorIfNotPristine { - liveFileNums := make(map[FileNum]struct{}) + liveFileNums := make(map[base.DiskFileNum]struct{}) d.mu.versions.addLiveFileNums(liveFileNums) if len(liveFileNums) != 0 { return nil, errors.Wrapf(ErrDBNotPristine, "dirname=%q", dirname) @@ -271,7 +285,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { // sequence number of the first batch that will be inserted. if !d.opts.ReadOnly { var entry *flushableEntry - d.mu.mem.mutable, entry = d.newMemTable(0 /* logNum */, d.mu.versions.atomic.logSeqNum) + d.mu.mem.mutable, entry = d.newMemTable(0 /* logNum */, d.mu.versions.logSeqNum.Load()) d.mu.mem.queue = append(d.mu.mem.queue, entry) } @@ -287,7 +301,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { } ls = append(ls, ls2...) } - providerSettings := objstorage.Settings{ + providerSettings := objstorageprovider.Settings{ Logger: opts.Logger, FS: opts.FS, FSDirName: dirname, @@ -296,13 +310,18 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { NoSyncOnClose: opts.NoSyncOnClose, BytesPerSync: opts.BytesPerSync, } - providerSettings.Shared.Storage = opts.Experimental.SharedStorage + providerSettings.Remote.StorageFactory = opts.Experimental.RemoteStorage + providerSettings.Remote.CreateOnShared = opts.Experimental.CreateOnShared + providerSettings.Remote.CreateOnSharedLocator = opts.Experimental.CreateOnSharedLocator + providerSettings.Remote.CacheSizeBytes = opts.Experimental.SecondaryCacheSizeBytes - d.objProvider, err = objstorage.Open(providerSettings) + d.objProvider, err = objstorageprovider.Open(providerSettings) if err != nil { return nil, err } + d.cleanupManager = openCleanupManager(opts, d.objProvider, d.onObsoleteTableDelete, d.getDeletionPacerInfo) + if manifestExists { curVersion := d.mu.versions.currentVersion() if err := checkConsistency(curVersion, dirname, d.objProvider); err != nil { @@ -331,21 +350,21 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { // Don't reuse any obsolete file numbers to avoid modifying an // ingested sstable's original external file. - if d.mu.versions.nextFileNum <= fn { - d.mu.versions.nextFileNum = fn + 1 + if d.mu.versions.nextFileNum <= fn.FileNum() { + d.mu.versions.nextFileNum = fn.FileNum() + 1 } switch ft { case fileTypeLog: - if fn >= d.mu.versions.minUnflushedLogNum { - logFiles = append(logFiles, fileNumAndName{fn, filename}) + if fn.FileNum() >= d.mu.versions.minUnflushedLogNum { + logFiles = append(logFiles, fileNumAndName{fn.FileNum(), filename}) } - if d.logRecycler.minRecycleLogNum <= fn { - d.logRecycler.minRecycleLogNum = fn + 1 + if d.logRecycler.minRecycleLogNum <= fn.FileNum() { + d.logRecycler.minRecycleLogNum = fn.FileNum() + 1 } case fileTypeOptions: - if previousOptionsFileNum < fn { - previousOptionsFileNum = fn + if previousOptionsFileNum < fn.FileNum() { + previousOptionsFileNum = fn.FileNum() previousOptionsFilename = filename } case fileTypeTemp, fileTypeOldTemp: @@ -362,6 +381,15 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { } } + // Ratchet d.mu.versions.nextFileNum ahead of all known objects in the + // objProvider. This avoids FileNum collisions with obsolete sstables. + objects := d.objProvider.List() + for _, obj := range objects { + if d.mu.versions.nextFileNum <= obj.DiskFileNum.FileNum() { + d.mu.versions.nextFileNum = obj.DiskFileNum.FileNum() + 1 + } + } + // Validate the most-recent OPTIONS file, if there is one. var strictWALTail bool if previousOptionsFilename != "" { @@ -387,11 +415,11 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { } toFlush = append(toFlush, flush...) d.mu.versions.markFileNumUsed(lf.num) - if d.mu.versions.atomic.logSeqNum < maxSeqNum { - d.mu.versions.atomic.logSeqNum = maxSeqNum + if d.mu.versions.logSeqNum.Load() < maxSeqNum { + d.mu.versions.logSeqNum.Store(maxSeqNum) } } - d.mu.versions.atomic.visibleSeqNum = d.mu.versions.atomic.logSeqNum + d.mu.versions.visibleSeqNum.Store(d.mu.versions.logSeqNum.Load()) if !d.opts.ReadOnly { // Create an empty .log file. @@ -417,8 +445,8 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { entry.readerUnrefLocked(true) } - newLogName := base.MakeFilepath(opts.FS, d.walDirname, fileTypeLog, newLogNum) - d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum, fileSize: 0}) + newLogName := base.MakeFilepath(opts.FS, d.walDirname, fileTypeLog, newLogNum.DiskFileNum()) + d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum.DiskFileNum(), fileSize: 0}) logFile, err := opts.FS.Create(newLogName) if err != nil { return nil, err @@ -469,7 +497,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { if !d.opts.ReadOnly { // Write the current options to disk. - d.optionsFileNum = d.mu.versions.getNextFileNum() + d.optionsFileNum = d.mu.versions.getNextFileNum().DiskFileNum() tmpPath := base.MakeFilepath(opts.FS, dirname, fileTypeTemp, d.optionsFileNum) optionsPath := base.MakeFilepath(opts.FS, dirname, fileTypeOptions, d.optionsFileNum) @@ -504,7 +532,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { if !d.opts.ReadOnly { d.scanObsoleteFiles(ls) - d.deleteObsoleteFiles(jobID, true /* waitForOngoing */) + d.deleteObsoleteFiles(jobID) } else { // All the log files are obsolete. d.mu.versions.metrics.WAL.Files = int64(len(logFiles)) @@ -613,9 +641,9 @@ func GetVersion(dir string, fs vfs.FS) (string, error) { // processed, reset version. This is because rocksdb often // writes multiple options files without deleting previous ones. // Otherwise, skip parsing this options file. - if fn > lastOptionsSeen { + if fn.FileNum() > lastOptionsSeen { version = "" - lastOptionsSeen = fn + lastOptionsSeen = fn.FileNum() } else { continue } @@ -677,8 +705,15 @@ func (d *DB) replayWAL( rr = record.NewReader(file, logNum) offset int64 // byte offset in rr lastFlushOffset int64 + keysReplayed int64 // number of keys replayed + batchesReplayed int64 // number of batches replayed ) + // TODO(jackson): This function is interspersed with panics, in addition to + // corruption error propagation. Audit them to ensure we're truly only + // panicking where the error points to Pebble bug and not user or + // hardware-induced corruption. + if d.opts.ReadOnly { // In read-only mode, we replay directly into the mutable memtable which will // never be flushed. @@ -725,7 +760,7 @@ func (d *DB) replayWAL( // TODO(bananabrick): See if we can use the actual base level here, // instead of using 1. c := newFlush(d.opts, d.mu.versions.currentVersion(), - 1 /* base level */, toFlush) + 1 /* base level */, toFlush, d.timeNow()) newVE, _, _, err := d.runCompaction(jobID, c) if err != nil { return errors.Wrapf(err, "running compaction during WAL replay") @@ -733,6 +768,11 @@ func (d *DB) replayWAL( ve.NewFiles = append(ve.NewFiles, newVE.NewFiles...) return nil } + defer func() { + if err != nil { + err = errors.WithDetailf(err, "replaying log %s, offset %d", logNum, offset) + } + }() for { offset = rr.Offset() @@ -765,26 +805,33 @@ func (d *DB) replayWAL( // Specify Batch.db so that Batch.SetRepr will compute Batch.memTableSize // which is used below. - b = Batch{db: d} + b = Batch{} + b.db = d b.SetRepr(buf.Bytes()) seqNum := b.SeqNum() maxSeqNum = seqNum + uint64(b.Count()) - + keysReplayed += int64(b.Count()) + batchesReplayed++ { br := b.Reader() - if kind, encodedFileNum, _, _ := br.Next(); kind == InternalKeyKindIngestSST { - fileNums := make([]FileNum, 0, b.Count()) + if kind, encodedFileNum, _, ok, err := br.Next(); err != nil { + return nil, 0, err + } else if ok && kind == InternalKeyKindIngestSST { + fileNums := make([]base.DiskFileNum, 0, b.Count()) addFileNum := func(encodedFileNum []byte) { fileNum, n := binary.Uvarint(encodedFileNum) if n <= 0 { panic("pebble: ingest sstable file num is invalid.") } - fileNums = append(fileNums, base.FileNum(fileNum)) + fileNums = append(fileNums, base.FileNum(fileNum).DiskFileNum()) } addFileNum(encodedFileNum) for i := 1; i < int(b.Count()); i++ { - kind, encodedFileNum, _, ok := br.Next() + kind, encodedFileNum, _, ok, err := br.Next() + if err != nil { + return nil, 0, err + } if kind != InternalKeyKindIngestSST { panic("pebble: invalid batch key kind.") } @@ -794,21 +841,41 @@ func (d *DB) replayWAL( addFileNum(encodedFileNum) } - if _, _, _, ok := br.Next(); ok { + if _, _, _, ok, err := br.Next(); err != nil { + return nil, 0, err + } else if ok { panic("pebble: invalid number of entries in batch.") } - paths := make([]string, len(fileNums)) + meta := make([]*fileMetadata, len(fileNums)) for i, n := range fileNums { - paths[i] = base.MakeFilepath(d.opts.FS, d.dirname, fileTypeTable, n) - } + var readable objstorage.Readable + objMeta, err := d.objProvider.Lookup(fileTypeTable, n) + if err != nil { + return nil, 0, errors.Wrap(err, "pebble: error when looking up ingested SSTs") + } + if objMeta.IsRemote() { + readable, err = d.objProvider.OpenForReading(context.TODO(), fileTypeTable, n, objstorage.OpenOptions{MustExist: true}) + if err != nil { + return nil, 0, errors.Wrap(err, "pebble: error when opening flushable ingest files") + } + } else { + path := base.MakeFilepath(d.opts.FS, d.dirname, fileTypeTable, n) + f, err := d.opts.FS.Open(path) + if err != nil { + return nil, 0, err + } - var meta []*manifest.FileMetadata - meta, _, err = ingestLoad( - d.opts, d.FormatMajorVersion(), paths, d.cacheID, fileNums, - ) - if err != nil { - return nil, 0, err + readable, err = sstable.NewSimpleReadable(f) + if err != nil { + return nil, 0, err + } + } + // NB: ingestLoad1 will close readable. + meta[i], err = ingestLoad1(d.opts, d.FormatMajorVersion(), readable, d.cacheID, n) + if err != nil { + return nil, 0, errors.Wrap(err, "pebble: error when loading flushable ingest files") + } } if uint32(len(meta)) != b.Count() { @@ -854,9 +921,10 @@ func (d *DB) replayWAL( d.opts, d.mu.versions.currentVersion(), 1, /* base level */ []*flushableEntry{entry}, + d.timeNow(), ) for _, file := range c.flushing[0].flushable.(*ingestedFlushable).files { - ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: 0, Meta: file}) + ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: 0, Meta: file.FileMetadata}) } } return toFlush, maxSeqNum, nil @@ -868,11 +936,14 @@ func (d *DB) replayWAL( // Make a copy of the data slice since it is currently owned by buf and will // be reused in the next iteration. b.data = append([]byte(nil), b.data...) - b.flushable = newFlushableBatch(&b, d.opts.Comparer) + b.flushable, err = newFlushableBatch(&b, d.opts.Comparer) + if err != nil { + return nil, 0, err + } entry := d.newFlushableEntry(b.flushable, logNum, b.SeqNum()) // Disable memory accounting by adding a reader ref that will never be // removed. - entry.readerRefs++ + entry.readerRefs.Add(1) if d.opts.ReadOnly { d.mu.mem.queue = append(d.mu.mem.queue, entry) // We added the flushable batch to the flushable to the queue. @@ -910,7 +981,10 @@ func (d *DB) replayWAL( } buf.Reset() } + + d.opts.Logger.Infof("[JOB %d] WAL file %s with log number %s stopped reading at offset: %d; replayed %d keys in %d batches", jobID, filename, logNum.String(), offset, keysReplayed, batchesReplayed) flushMem() + // mem is nil here. if !d.opts.ReadOnly { err = updateVE() @@ -990,7 +1064,7 @@ func Peek(dirname string, fs vfs.FS) (*DBDesc, error) { // LockDirectory may be used to expand the critical section protected by the // database lock to include setup before the call to Open. func LockDirectory(dirname string, fs vfs.FS) (*Lock, error) { - fileLock, err := fs.Lock(base.MakeFilepath(fs, dirname, fileTypeLock, 0)) + fileLock, err := fs.Lock(base.MakeFilepath(fs, dirname, fileTypeLock, base.FileNum(0).DiskFileNum())) if err != nil { return nil, err } @@ -1068,28 +1142,42 @@ func IsCorruptionError(err error) bool { return errors.Is(err, base.ErrCorruption) } -func checkConsistency(v *manifest.Version, dirname string, objProvider *objstorage.Provider) error { +func checkConsistency(v *manifest.Version, dirname string, objProvider objstorage.Provider) error { var buf bytes.Buffer var args []interface{} + dedup := make(map[base.DiskFileNum]struct{}) for level, files := range v.Levels { iter := files.Iter() for f := iter.First(); f != nil; f = iter.Next() { - meta, err := objProvider.Lookup(base.FileTypeTable, f.FileNum) + backingState := f.FileBacking + if _, ok := dedup[backingState.DiskFileNum]; ok { + continue + } + dedup[backingState.DiskFileNum] = struct{}{} + fileNum := backingState.DiskFileNum + fileSize := backingState.Size + // We allow foreign objects to have a mismatch between sizes. This is + // because we might skew the backing size stored by our objprovider + // to prevent us from over-prioritizing this file for compaction. + meta, err := objProvider.Lookup(base.FileTypeTable, fileNum) var size int64 if err == nil { + if objProvider.IsSharedForeign(meta) { + continue + } size, err = objProvider.Size(meta) } if err != nil { buf.WriteString("L%d: %s: %v\n") - args = append(args, errors.Safe(level), errors.Safe(f.FileNum), err) + args = append(args, errors.Safe(level), errors.Safe(fileNum), err) continue } - if size != int64(f.Size) { + if size != int64(fileSize) { buf.WriteString("L%d: %s: object size mismatch (%s): %d (disk) != %d (MANIFEST)\n") - args = append(args, errors.Safe(level), errors.Safe(f.FileNum), objProvider.Path(meta), - errors.Safe(size), errors.Safe(f.Size)) + args = append(args, errors.Safe(level), errors.Safe(fileNum), objProvider.Path(meta), + errors.Safe(size), errors.Safe(fileSize)) continue } } diff --git a/vendor/github.com/cockroachdb/pebble/options.go b/vendor/github.com/cockroachdb/pebble/options.go index 895276c..f777daa 100644 --- a/vendor/github.com/cockroachdb/pebble/options.go +++ b/vendor/github.com/cockroachdb/pebble/options.go @@ -17,8 +17,10 @@ import ( "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/cache" "github.com/cockroachdb/pebble/internal/humanize" + "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/manifest" - "github.com/cockroachdb/pebble/objstorage/shared" + "github.com/cockroachdb/pebble/objstorage/remote" + "github.com/cockroachdb/pebble/rangekey" "github.com/cockroachdb/pebble/sstable" "github.com/cockroachdb/pebble/vfs" ) @@ -117,11 +119,24 @@ type IterOptions struct { // false to skip scanning. This function must be thread-safe since the same // function can be used by multiple iterators, if the iterator is cloned. TableFilter func(userProps map[string]string) bool + // SkipPoint may be used to skip over point keys that don't match an + // arbitrary predicate during iteration. If set, the Iterator invokes + // SkipPoint for keys encountered. If SkipPoint returns true, the iterator + // will skip the key without yielding it to the iterator operation in + // progress. + // + // SkipPoint must be a pure function and always return the same result when + // provided the same arguments. The iterator may call SkipPoint multiple + // times for the same user key. + SkipPoint func(userKey []byte) bool // PointKeyFilters can be used to avoid scanning tables and blocks in tables - // when iterating over point keys. It is requires that this slice is sorted in - // increasing order of the BlockPropertyFilter.ShortID. This slice represents - // an intersection across all filters, i.e., all filters must indicate that the - // block is relevant. + // when iterating over point keys. This slice represents an intersection + // across all filters, i.e., all filters must indicate that the block is + // relevant. + // + // Performance note: When len(PointKeyFilters) > 0, the caller should ensure + // that cap(PointKeyFilters) is at least len(PointKeyFilters)+1. This helps + // avoid allocations in Pebble internal code that mutates the slice. PointKeyFilters []BlockPropertyFilter // RangeKeyFilters can be usefd to avoid scanning tables and blocks in tables // when iterating over range keys. The same requirements that apply to @@ -180,6 +195,10 @@ type IterOptions struct { level manifest.Level // disableLazyCombinedIteration is an internal testing option. disableLazyCombinedIteration bool + // snapshotForHideObsoletePoints is specified for/by levelIter when opening + // files and is used to decide whether to hide obsolete points. A value of 0 + // implies obsolete points should not be hidden. + snapshotForHideObsoletePoints uint64 // NB: If adding new Options, you must account for them in iterator // construction and Iterator.SetOptions. @@ -222,6 +241,38 @@ func (o *IterOptions) getLogger() Logger { return o.logger } +// SpanIterOptions creates a SpanIterOptions from this IterOptions. +func (o *IterOptions) SpanIterOptions() keyspan.SpanIterOptions { + if o == nil { + return keyspan.SpanIterOptions{} + } + return keyspan.SpanIterOptions{ + RangeKeyFilters: o.RangeKeyFilters, + } +} + +// scanInternalOptions is similar to IterOptions, meant for use with +// scanInternalIterator. +type scanInternalOptions struct { + IterOptions + + visitPointKey func(key *InternalKey, value LazyValue, iterInfo IteratorLevel) error + visitRangeDel func(start, end []byte, seqNum uint64) error + visitRangeKey func(start, end []byte, keys []rangekey.Key) error + visitSharedFile func(sst *SharedSSTMeta) error + + // skipSharedLevels skips levels that are shareable (level >= + // sharedLevelStart). + skipSharedLevels bool + + // includeObsoleteKeys specifies whether keys shadowed by newer internal keys + // are exposed. If false, only one internal key per user key is exposed. + includeObsoleteKeys bool + + // rateLimitFunc is used to limit the amount of bytes read per second. + rateLimitFunc func(key *InternalKey, value LazyValue) error +} + // RangeKeyMasking configures automatic hiding of point keys by range keys. A // non-nil Suffix enables range-key masking. When enabled, range keys with // suffixes ≥ Suffix behave as masks. All point keys that are contained within a @@ -497,18 +548,12 @@ type Options struct { // concurrent compaction is added. This works "on top" of // L0CompactionConcurrency, so the higher of the count of compaction // concurrency slots as determined by the two options is chosen. - CompactionDebtConcurrency int + CompactionDebtConcurrency uint64 - // MinDeletionRate is the minimum number of bytes per second that would - // be deleted. Deletion pacing is used to slow down deletions when - // compactions finish up or readers close, and newly-obsolete files need - // cleaning up. Deleting lots of files at once can cause disk latency to - // go up on some SSDs, which this functionality guards against. This is a - // minimum as the maximum is theoretically unlimited; pacing is disabled - // when there are too many obsolete files relative to live bytes, or - // there isn't enough disk space available. Setting this to 0 disables - // deletion pacing, which is also the default. - MinDeletionRate int + // IngestSplit, if it returns true, allows for ingest-time splitting of + // existing sstables into two virtual sstables to allow ingestion sstables to + // slot into a lower level than they otherwise would have. + IngestSplit func() bool // ReadCompactionRate controls the frequency of read triggered // compactions by adjusting `AllowedSeeks` in manifest.FileMetadata: @@ -573,10 +618,10 @@ type Options struct { // desired size of each level of the LSM. Defaults to 10. LevelMultiplier int - // MultiLevelCompactionHueristic determines whether to add an additional + // MultiLevelCompactionHeuristic determines whether to add an additional // level to a conventional two level compaction. If nil, a multilevel // compaction will never get triggered. - MultiLevelCompactionHueristic MultiLevelHeuristic + MultiLevelCompactionHeuristic MultiLevelHeuristic // MaxWriterConcurrency is used to indicate the maximum number of // compression workers the compression queue is allowed to use. If @@ -597,8 +642,11 @@ type Options struct { CPUWorkPermissionGranter CPUWorkPermissionGranter // EnableValueBlocks is used to decide whether to enable writing - // TableFormatPebblev3 sstables. WARNING: do not return true yet, since - // support for TableFormatPebblev3 is incomplete and not production ready. + // TableFormatPebblev3 sstables. This setting is only respected by a + // specific subset of format major versions: FormatSSTableValueBlocks, + // FormatFlushableIngest and FormatPrePebblev1MarkedCompacted. In lower + // format major versions, value blocks are never enabled. In higher + // format major versions, value blocks are always enabled. EnableValueBlocks func() bool // ShortAttributeExtractor is used iff EnableValueBlocks() returns true @@ -607,18 +655,24 @@ type Options struct { ShortAttributeExtractor ShortAttributeExtractor // RequiredInPlaceValueBound specifies an optional span of user key - // prefixes for which the values must be stored with the key. This is - // useful for statically known exclusions to value separation. In - // CockroachDB, this will be used for the lock table key space that has - // non-empty suffixes, but those locks don't represent actual MVCC - // versions (the suffix ordering is arbitrary). We will also need to add - // support for dynamically configured exclusions (we want the default to - // be to allow Pebble to decide whether to separate the value or not, - // hence this is structured as exclusions), for example, for users of - // CockroachDB to dynamically exclude certain tables. + // prefixes that are not-MVCC, but have a suffix. For these the values + // must be stored with the key, since the concept of "older versions" is + // not defined. It is also useful for statically known exclusions to value + // separation. In CockroachDB, this will be used for the lock table key + // space that has non-empty suffixes, but those locks don't represent + // actual MVCC versions (the suffix ordering is arbitrary). We will also + // need to add support for dynamically configured exclusions (we want the + // default to be to allow Pebble to decide whether to separate the value + // or not, hence this is structured as exclusions), for example, for users + // of CockroachDB to dynamically exclude certain tables. // // Any change in exclusion behavior takes effect only on future written // sstables, and does not start rewriting existing sstables. + // + // Even ignoring changes in this setting, exclusions are interpreted as a + // guidance by Pebble, and not necessarily honored. Specifically, user + // keys with multiple Pebble-versions *may* have the older versions stored + // in value blocks. RequiredInPlaceValueBound UserKeyPrefixBound // DisableIngestAsFlushable disables lazy ingestion of sstables through @@ -626,15 +680,49 @@ type Options struct { // major version is at least `FormatFlushableIngest`. DisableIngestAsFlushable func() bool - // SharedStorage is a second FS-like storage medium that can be shared - // between multiple Pebble instances. It is used to store sstables only, and - // is managed by objstorage.Provider. Each sstable might only be written to - // by one Pebble instance, but other Pebble instances can possibly read the - // same files if they have the path to get to them. The pebble instance that - // wrote a file should not delete it if other Pebble instances are known to - // be reading this file. This FS is expected to have slower read/write - // performance than the default FS above. - SharedStorage shared.Storage + // RemoteStorage enables use of remote storage (e.g. S3) for storing + // sstables. Setting this option enables use of CreateOnShared option and + // allows ingestion of external files. + RemoteStorage remote.StorageFactory + + // If CreateOnShared is non-zero, new sstables are created on remote storage + // (using CreateOnSharedLocator and with the appropriate + // CreateOnSharedStrategy). These sstables can be shared between different + // Pebble instances; the lifecycle of such objects is managed by the + // remote.Storage constructed by options.RemoteStorage. + // + // Can only be used when RemoteStorage is set (and recognizes + // CreateOnSharedLocator). + CreateOnShared remote.CreateOnSharedStrategy + CreateOnSharedLocator remote.Locator + + // CacheSizeBytesBytes is the size of the on-disk block cache for objects + // on shared storage in bytes. If it is 0, no cache is used. + SecondaryCacheSizeBytes int64 + + // IneffectualPointDeleteCallback is called in compactions/flushes if any + // single delete is being elided without deleting a point set/merge. + IneffectualSingleDeleteCallback func(userKey []byte) + + // SingleDeleteInvariantViolationCallback is called in compactions/flushes if any + // single delete has consumed a Set/Merge, and there is another immediately older + // Set/SetWithDelete/Merge. The user of Pebble has violated the invariant under + // which SingleDelete can be used correctly. + // + // Consider the sequence SingleDelete#3, Set#2, Set#1. There are three + // ways some of these keys can first meet in a compaction. + // + // - All 3 keys in the same compaction: this callback will detect the + // violation. + // + // - SingleDelete#3, Set#2 meet in a compaction first: Both keys will + // disappear. The violation will not be detected, and the DB will have + // Set#1 which is likely incorrect (from the user's perspective). + // + // - Set#2, Set#1 meet in a compaction first: The output will be Set#2, + // which will later be consumed by SingleDelete#3. The violation will + // not be detected and the DB will be correct. + SingleDeleteInvariantViolationCallback func(userKey []byte) } // Filters is a map from filter policy name to filter policy. It is used for @@ -749,12 +837,18 @@ type Options struct { // writing the contents of the old one in the // background. MemTableStopWritesThreshold places a hard limit on the size of // the queued MemTables. - MemTableSize int + // + // The default value is 4MB. + MemTableSize uint64 - // Hard limit on the size of queued of MemTables. Writes are stopped when the - // sum of the queued memtable sizes exceeds - // MemTableStopWritesThreshold*MemTableSize. This value should be at least 2 - // or writes will stop whenever a MemTable is being flushed. + // Hard limit on the number of queued of MemTables. Writes are stopped when + // the sum of the queued memtable sizes exceeds: + // MemTableStopWritesThreshold * MemTableSize. + // + // This value should be at least 2 or writes will stop whenever a MemTable is + // being flushed. + // + // The default value is 2. MemTableStopWritesThreshold int // Merger defines the associative merge operation to use for merging values @@ -839,6 +933,20 @@ type Options struct { // changing options dynamically? WALMinSyncInterval func() time.Duration + // TargetByteDeletionRate is the rate (in bytes per second) at which sstable file + // deletions are limited to (under normal circumstances). + // + // Deletion pacing is used to slow down deletions when compactions finish up + // or readers close and newly-obsolete files need cleaning up. Deleting lots + // of files at once can cause disk latency to go up on some SSDs, which this + // functionality guards against. + // + // This value is only a best-effort target; the effective rate can be + // higher if deletions are falling behind or disk space is running low. + // + // Setting this to 0 disables deletion pacing, which is also the default. + TargetByteDeletionRate int + // private options are only used by internal tests or are used internally // for facilitating upgrade paths of unconfigurable functionality. private struct { @@ -872,6 +980,10 @@ type Options struct { // A private option to disable stats collection. disableTableStats bool + // testingAlwaysWaitForCleanup is set by some tests to force waiting for + // obsolete file deletion (to make events deterministic). + testingAlwaysWaitForCleanup bool + // fsCloser holds a closer that should be invoked after a DB using these // Options is closed. This is used to automatically stop the // long-running goroutine associated with the disk-health-checking FS. @@ -983,7 +1095,7 @@ func (o *Options) EnsureDefaults() *Options { o.MaxOpenFiles = 1000 } if o.MemTableSize <= 0 { - o.MemTableSize = 4 << 20 + o.MemTableSize = 4 << 20 // 4 MB } if o.MemTableStopWritesThreshold <= 0 { o.MemTableStopWritesThreshold = 2 @@ -1024,8 +1136,8 @@ func (o *Options) EnsureDefaults() *Options { if o.Experimental.CPUWorkPermissionGranter == nil { o.Experimental.CPUWorkPermissionGranter = defaultCPUWorkGranter{} } - if o.Experimental.MultiLevelCompactionHueristic == nil { - o.Experimental.MultiLevelCompactionHueristic = NoMultiLevel{} + if o.Experimental.MultiLevelCompactionHeuristic == nil { + o.Experimental.MultiLevelCompactionHeuristic = WriteAmpHeuristic{} } o.initMaps() @@ -1144,7 +1256,7 @@ func (o *Options) String() string { fmt.Fprintf(&buf, " max_open_files=%d\n", o.MaxOpenFiles) fmt.Fprintf(&buf, " mem_table_size=%d\n", o.MemTableSize) fmt.Fprintf(&buf, " mem_table_stop_writes_threshold=%d\n", o.MemTableStopWritesThreshold) - fmt.Fprintf(&buf, " min_deletion_rate=%d\n", o.Experimental.MinDeletionRate) + fmt.Fprintf(&buf, " min_deletion_rate=%d\n", o.TargetByteDeletionRate) fmt.Fprintf(&buf, " merger=%s\n", o.Merger.Name) fmt.Fprintf(&buf, " read_compaction_rate=%d\n", o.Experimental.ReadCompactionRate) fmt.Fprintf(&buf, " read_sampling_multiplier=%d\n", o.Experimental.ReadSamplingMultiplier) @@ -1165,6 +1277,8 @@ func (o *Options) String() string { fmt.Fprintf(&buf, " wal_bytes_per_sync=%d\n", o.WALBytesPerSync) fmt.Fprintf(&buf, " max_writer_concurrency=%d\n", o.Experimental.MaxWriterConcurrency) fmt.Fprintf(&buf, " force_writer_parallelism=%t\n", o.Experimental.ForceWriterParallelism) + fmt.Fprintf(&buf, " secondary_cache_size_bytes=%d\n", o.Experimental.SecondaryCacheSizeBytes) + fmt.Fprintf(&buf, " create_on_shared=%d\n", o.Experimental.CreateOnShared) // Private options. // @@ -1322,7 +1436,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error { } } case "compaction_debt_concurrency": - o.Experimental.CompactionDebtConcurrency, err = strconv.Atoi(value) + o.Experimental.CompactionDebtConcurrency, err = strconv.ParseUint(value, 10, 64) case "delete_range_flush_delay": // NB: This is a deprecated serialization of the // `flush_delay_delete_range`. @@ -1354,7 +1468,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error { // version is valid right here. var v uint64 v, err = strconv.ParseUint(value, 10, 64) - if vers := FormatMajorVersion(v); vers > FormatNewest || vers == FormatDefault { + if vers := FormatMajorVersion(v); vers > internalFormatNewest || vers == FormatDefault { err = errors.Newf("unknown format major version %d", o.FormatMajorVersion) } if err == nil { @@ -1387,14 +1501,14 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error { case "max_open_files": o.MaxOpenFiles, err = strconv.Atoi(value) case "mem_table_size": - o.MemTableSize, err = strconv.Atoi(value) + o.MemTableSize, err = strconv.ParseUint(value, 10, 64) case "mem_table_stop_writes_threshold": o.MemTableStopWritesThreshold, err = strconv.Atoi(value) case "min_compaction_rate": // Do nothing; option existed in older versions of pebble, and // may be meaningful again eventually. case "min_deletion_rate": - o.Experimental.MinDeletionRate, err = strconv.Atoi(value) + o.TargetByteDeletionRate, err = strconv.Atoi(value) case "min_flush_rate": // Do nothing; option existed in older versions of pebble, and // may be meaningful again eventually. @@ -1438,6 +1552,12 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error { o.Experimental.MaxWriterConcurrency, err = strconv.Atoi(value) case "force_writer_parallelism": o.Experimental.ForceWriterParallelism, err = strconv.ParseBool(value) + case "secondary_cache_size_bytes": + o.Experimental.SecondaryCacheSizeBytes, err = strconv.ParseInt(value, 10, 64) + case "create_on_shared": + var createOnSharedInt int64 + createOnSharedInt, err = strconv.ParseInt(value, 10, 64) + o.Experimental.CreateOnShared = remote.CreateOnSharedStrategy(createOnSharedInt) default: if hooks != nil && hooks.SkipUnknown != nil && hooks.SkipUnknown(section+"."+key, value) { return nil @@ -1568,15 +1688,15 @@ func (o *Options) Validate() error { } if uint64(o.MemTableSize) >= maxMemTableSize { fmt.Fprintf(&buf, "MemTableSize (%s) must be < %s\n", - humanize.Uint64(uint64(o.MemTableSize)), humanize.Uint64(maxMemTableSize)) + humanize.Bytes.Uint64(uint64(o.MemTableSize)), humanize.Bytes.Uint64(maxMemTableSize)) } if o.MemTableStopWritesThreshold < 2 { fmt.Fprintf(&buf, "MemTableStopWritesThreshold (%d) must be >= 2\n", o.MemTableStopWritesThreshold) } - if o.FormatMajorVersion > FormatNewest { + if o.FormatMajorVersion > internalFormatNewest { fmt.Fprintf(&buf, "FormatMajorVersion (%d) must be <= %d\n", - o.FormatMajorVersion, FormatNewest) + o.FormatMajorVersion, internalFormatNewest) } if o.TableCache != nil && o.Cache != o.TableCache.cache { fmt.Fprintf(&buf, "underlying cache in the TableCache and the Cache dont match\n") @@ -1596,6 +1716,7 @@ func (o *Options) MakeReaderOptions() sstable.ReaderOptions { readerOpts.Comparer = o.Comparer readerOpts.Filters = o.Filters if o.Merger != nil { + readerOpts.Merge = o.Merger.Merge readerOpts.MergerName = o.Merger.Name } readerOpts.LoggerAndTracer = o.LoggerAndTracer @@ -1620,6 +1741,9 @@ func (o *Options) MakeWriterOptions(level int, format sstable.TableFormat) sstab if format >= sstable.TableFormatPebblev3 { writerOpts.ShortAttributeExtractor = o.Experimental.ShortAttributeExtractor writerOpts.RequiredInPlaceValueBound = o.Experimental.RequiredInPlaceValueBound + if format >= sstable.TableFormatPebblev4 && level == numLevels-1 { + writerOpts.WritingToLowestLevel = true + } } levelOpts := o.Level(level) writerOpts.BlockRestartInterval = levelOpts.BlockRestartInterval diff --git a/vendor/github.com/cockroachdb/pebble/pacer.go b/vendor/github.com/cockroachdb/pebble/pacer.go index 7546e56..a959ff4 100644 --- a/vendor/github.com/cockroachdb/pebble/pacer.go +++ b/vendor/github.com/cockroachdb/pebble/pacer.go @@ -7,30 +7,11 @@ package pebble import ( "sync" "time" - - "github.com/cockroachdb/errors" - "github.com/cockroachdb/pebble/internal/rate" ) -var nilPacer = &noopPacer{} - -type limiter interface { - DelayN(now time.Time, n int) time.Duration - AllowN(now time.Time, n int) bool - Burst() int -} - -// pacer is the interface for flush and compaction rate limiters. The rate limiter -// is possible applied on each iteration step of a flush or compaction. This is to -// limit background IO usage so that it does not contend with foreground traffic. -type pacer interface { - maybeThrottle(bytesIterated uint64) error - - reportDeletion(bytesToDelete uint64) -} - // deletionPacerInfo contains any info from the db necessary to make deletion -// pacing decisions. +// pacing decisions (to limit background IO usage so that it does not contend +// with foreground traffic). type deletionPacerInfo struct { freeBytes uint64 obsoleteBytes uint64 @@ -43,12 +24,17 @@ type deletionPacerInfo struct { // negatively impacted if too many blocks are deleted very quickly, so this // mechanism helps mitigate that. type deletionPacer struct { - limiter limiter - freeSpaceThreshold uint64 - obsoleteBytesMaxRatio float64 - targetByteDeletionRate int64 + // If there are less than freeSpaceThreshold bytes of free space on + // disk, increase the pace of deletions such that we delete enough bytes to + // get back to the threshold within the freeSpaceTimeframe. + freeSpaceThreshold uint64 + freeSpaceTimeframe time.Duration - getInfo func() deletionPacerInfo + // If the ratio of obsolete bytes to live bytes is greater than + // obsoleteBytesMaxRatio, increase the pace of deletions such that we delete + // enough bytes to get back to the threshold within the obsoleteBytesTimeframe. + obsoleteBytesMaxRatio float64 + obsoleteBytesTimeframe time.Duration mu struct { sync.Mutex @@ -57,110 +43,93 @@ type deletionPacer struct { // deletion rate to match the pace of deletions. history history } + + targetByteDeletionRate int64 + + getInfo func() deletionPacerInfo } const deletePacerHistory = 5 * time.Minute // newDeletionPacer instantiates a new deletionPacer for use when deleting -// obsolete files. The limiter passed in must be a singleton shared across this -// pebble instance. +// obsolete files. +// +// targetByteDeletionRate is the rate (in bytes/sec) at which we want to +// normally limit deletes (when we are not falling behind or running out of +// space). A value of 0.0 disables pacing. func newDeletionPacer( - targetByteDeletionRate int64, getInfo func() deletionPacerInfo, + now time.Time, targetByteDeletionRate int64, getInfo func() deletionPacerInfo, ) *deletionPacer { d := &deletionPacer{ - limiter: rate.NewLimiter(rate.Limit(targetByteDeletionRate), int(targetByteDeletionRate)), - - // If there are less than freeSpaceThreshold bytes of free space on - // disk, do not pace deletions at all. freeSpaceThreshold: 16 << 30, // 16 GB - // If the ratio of obsolete bytes to live bytes is greater than - // obsoleteBytesMaxRatio, do not pace deletions at all. - obsoleteBytesMaxRatio: 0.20, + freeSpaceTimeframe: 10 * time.Second, + + obsoleteBytesMaxRatio: 0.20, + obsoleteBytesTimeframe: 5 * time.Minute, targetByteDeletionRate: targetByteDeletionRate, - - getInfo: getInfo, + getInfo: getInfo, } - d.mu.history.Init(time.Now(), deletePacerHistory) + d.mu.history.Init(now, deletePacerHistory) return d } -// limit applies rate limiting if the current free disk space is more than -// freeSpaceThreshold, and the ratio of obsolete to live bytes is less than -// obsoleteBytesMaxRatio. -func (p *deletionPacer) limit(amount uint64, info deletionPacerInfo) error { - obsoleteBytesRatio := float64(1.0) - if info.liveBytes > 0 { - obsoleteBytesRatio = float64(info.obsoleteBytes) / float64(info.liveBytes) - } - paceDeletions := info.freeBytes > p.freeSpaceThreshold && - obsoleteBytesRatio < p.obsoleteBytesMaxRatio - - p.mu.Lock() - if historyRate := p.mu.history.Sum(time.Now()) / int64(deletePacerHistory/time.Second); historyRate > p.targetByteDeletionRate { - // Deletions have been happening at a rate higher than the target rate; we - // want to speed up deletions so they match the historic rate. We do this by - // decreasing the amount accordingly. - amount = uint64(float64(p.targetByteDeletionRate) / float64(historyRate) * float64(amount)) - } - p.mu.Unlock() - - if paceDeletions { - burst := p.limiter.Burst() - for amount > uint64(burst) { - d := p.limiter.DelayN(time.Now(), burst) - if d == rate.InfDuration { - return errors.Errorf("pacing failed") - } - time.Sleep(d) - amount -= uint64(burst) - } - d := p.limiter.DelayN(time.Now(), int(amount)) - if d == rate.InfDuration { - return errors.Errorf("pacing failed") - } - time.Sleep(d) - } else { - burst := p.limiter.Burst() - for amount > uint64(burst) { - // AllowN will subtract burst if there are enough tokens available, - // else leave the tokens untouched. That is, we are making a - // best-effort to account for this activity in the limiter, but by - // ignoring the return value, we do the activity instantaneously - // anyway. - p.limiter.AllowN(time.Now(), burst) - amount -= uint64(burst) - } - p.limiter.AllowN(time.Now(), int(amount)) - } - return nil -} - -// maybeThrottle slows down a deletion of this file if it's faster than -// opts.Experimental.MinDeletionRate. -func (p *deletionPacer) maybeThrottle(bytesToDelete uint64) error { - return p.limit(bytesToDelete, p.getInfo()) -} - -// reportDeletion is used to report a deletion to the pacer. The pacer uses it +// ReportDeletion is used to report a deletion to the pacer. The pacer uses it // to keep track of the recent rate of deletions and potentially increase the // deletion rate accordingly. // -// reportDeletion is thread-safe. -func (p *deletionPacer) reportDeletion(bytesToDelete uint64) { +// ReportDeletion is thread-safe. +func (p *deletionPacer) ReportDeletion(now time.Time, bytesToDelete uint64) { p.mu.Lock() defer p.mu.Unlock() - p.mu.history.Add(time.Now(), int64(bytesToDelete)) + p.mu.history.Add(now, int64(bytesToDelete)) } -type noopPacer struct{} +// PacingDelay returns the recommended pacing wait time (in seconds) for +// deleting the given number of bytes. +// +// PacingDelay is thread-safe. +func (p *deletionPacer) PacingDelay(now time.Time, bytesToDelete uint64) (waitSeconds float64) { + if p.targetByteDeletionRate == 0 { + // Pacing disabled. + return 0.0 + } -func (p *noopPacer) maybeThrottle(_ uint64) error { - return nil + baseRate := float64(p.targetByteDeletionRate) + // If recent deletion rate is more than our target, use that so that we don't + // fall behind. + historicRate := func() float64 { + p.mu.Lock() + defer p.mu.Unlock() + return float64(p.mu.history.Sum(now)) / deletePacerHistory.Seconds() + }() + if historicRate > baseRate { + baseRate = historicRate + } + + // Apply heuristics to increase the deletion rate. + var extraRate float64 + info := p.getInfo() + if info.freeBytes <= p.freeSpaceThreshold { + // Increase the rate so that we can free up enough bytes within the timeframe. + extraRate = float64(p.freeSpaceThreshold-info.freeBytes) / p.freeSpaceTimeframe.Seconds() + } + if info.liveBytes == 0 { + // We don't know the obsolete bytes ratio. Disable pacing altogether. + return 0.0 + } + obsoleteBytesRatio := float64(info.obsoleteBytes) / float64(info.liveBytes) + if obsoleteBytesRatio >= p.obsoleteBytesMaxRatio { + // Increase the rate so that we can free up enough bytes within the timeframe. + r := (obsoleteBytesRatio - p.obsoleteBytesMaxRatio) * float64(info.liveBytes) / p.obsoleteBytesTimeframe.Seconds() + if extraRate < r { + extraRate = r + } + } + + return float64(bytesToDelete) / (baseRate + extraRate) } -func (p *noopPacer) reportDeletion(_ uint64) {} - // history is a helper used to keep track of the recent history of a set of // data points (in our case deleted bytes), at limited granularity. // Specifically, we split the desired timeframe into 100 "epochs" and all times diff --git a/vendor/github.com/cockroachdb/pebble/range_keys.go b/vendor/github.com/cockroachdb/pebble/range_keys.go index 30549d5..13b5822 100644 --- a/vendor/github.com/cockroachdb/pebble/range_keys.go +++ b/vendor/github.com/cockroachdb/pebble/range_keys.go @@ -17,7 +17,7 @@ import ( func (i *Iterator) constructRangeKeyIter() { i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init( &i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound, - &i.hasPrefix, &i.prefixOrFullSeekKey, true /* onlySets */, &i.rangeKey.rangeKeyBuffers.internal) + &i.hasPrefix, &i.prefixOrFullSeekKey, false /* internalKeys */, &i.rangeKey.rangeKeyBuffers.internal) // If there's an indexed batch with range keys, include it. if i.batch != nil { @@ -36,19 +36,24 @@ func (i *Iterator) constructRangeKeyIter() { } // Next are the flushables: memtables and large batches. - for j := len(i.readState.memtables) - 1; j >= 0; j-- { - mem := i.readState.memtables[j] - // We only need to read from memtables which contain sequence numbers older - // than seqNum. - if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum { - continue - } - if rki := mem.newRangeKeyIter(&i.opts); rki != nil { - i.rangeKey.iterConfig.AddLevel(rki) + if i.readState != nil { + for j := len(i.readState.memtables) - 1; j >= 0; j-- { + mem := i.readState.memtables[j] + // We only need to read from memtables which contain sequence numbers older + // than seqNum. + if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum { + continue + } + if rki := mem.newRangeKeyIter(&i.opts); rki != nil { + i.rangeKey.iterConfig.AddLevel(rki) + } } } - current := i.readState.current + current := i.version + if current == nil { + current = i.readState.current + } // Next are the file levels: L0 sub-levels followed by lower levels. // // Add file-specific iterators for L0 files containing range keys. This is less @@ -63,8 +68,7 @@ func (i *Iterator) constructRangeKeyIter() { // around Key Trailer order. iter := current.RangeKeyLevels[0].Iter() for f := iter.Last(); f != nil; f = iter.Prev() { - spanIterOpts := &keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters} - spanIter, err := i.newIterRangeKey(f, spanIterOpts) + spanIter, err := i.newIterRangeKey(f, i.opts.SpanIterOptions()) if err != nil { i.rangeKey.iterConfig.AddLevel(&errorKeyspanIter{err: err}) continue @@ -78,7 +82,7 @@ func (i *Iterator) constructRangeKeyIter() { continue } li := i.rangeKey.iterConfig.NewLevelIter() - spanIterOpts := keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters} + spanIterOpts := i.opts.SpanIterOptions() li.Init(spanIterOpts, i.cmp, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(), manifest.Level(level), manifest.KeyTypeRange) i.rangeKey.iterConfig.AddLevel(li) @@ -366,24 +370,24 @@ func (m *rangeKeyMasking) Intersects(prop []byte) (bool, error) { // KeyIsWithinLowerBound implements the limitedBlockPropertyFilter interface // defined in the sstable package. It's used to restrict the masking block // property filter to only applying within the bounds of the active range key. -func (m *rangeKeyMasking) KeyIsWithinLowerBound(ik *InternalKey) bool { +func (m *rangeKeyMasking) KeyIsWithinLowerBound(key []byte) bool { // Invariant: m.maskSpan != nil // - // The provided `ik` is an inclusive lower bound of the block we're + // The provided `key` is an inclusive lower bound of the block we're // considering skipping. - return m.cmp(m.maskSpan.Start, ik.UserKey) <= 0 + return m.cmp(m.maskSpan.Start, key) <= 0 } // KeyIsWithinUpperBound implements the limitedBlockPropertyFilter interface // defined in the sstable package. It's used to restrict the masking block // property filter to only applying within the bounds of the active range key. -func (m *rangeKeyMasking) KeyIsWithinUpperBound(ik *InternalKey) bool { +func (m *rangeKeyMasking) KeyIsWithinUpperBound(key []byte) bool { // Invariant: m.maskSpan != nil // - // The provided `ik` is an *inclusive* upper bound of the block we're + // The provided `key` is an *inclusive* upper bound of the block we're // considering skipping, so the range key's end must be strictly greater // than the block bound for the block to be within bounds. - return m.cmp(m.maskSpan.End, ik.UserKey) > 0 + return m.cmp(m.maskSpan.End, key) > 0 } // lazyCombinedIter implements the internalIterator interface, wrapping a @@ -512,7 +516,11 @@ func (i *lazyCombinedIter) initCombinedIteration( // Initialize the Iterator's interleaving iterator. i.parent.rangeKey.iiter.Init( &i.parent.comparer, i.parent.pointIter, i.parent.rangeKey.rangeKeyIter, - &i.parent.rangeKeyMasking, i.parent.opts.LowerBound, i.parent.opts.UpperBound) + keyspan.InterleavingIterOpts{ + Mask: &i.parent.rangeKeyMasking, + LowerBound: i.parent.opts.LowerBound, + UpperBound: i.parent.opts.UpperBound, + }) // Set the parent's primary iterator to point to the combined, interleaving // iterator that's now initialized with our current state. diff --git a/vendor/github.com/cockroachdb/pebble/rangekey/rangekey.go b/vendor/github.com/cockroachdb/pebble/rangekey/rangekey.go new file mode 100644 index 0000000..93e7fbe --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/rangekey/rangekey.go @@ -0,0 +1,33 @@ +// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +// Package rangekey provides functionality for working with range keys. +package rangekey + +import ( + "github.com/cockroachdb/pebble/internal/keyspan" + "github.com/cockroachdb/pebble/internal/rangekey" + "github.com/cockroachdb/pebble/sstable" +) + +// Fragmenter exports the keyspan.Fragmenter type. +type Fragmenter = keyspan.Fragmenter + +// Key exports the keyspan.Key type. +type Key = keyspan.Key + +// Span exports the keyspan.Span type. +type Span = keyspan.Span + +// IsRangeKey returns if this InternalKey is a range key. Alias for +// rangekey.IsRangeKey. +func IsRangeKey(ik sstable.InternalKey) bool { + return rangekey.IsRangeKey(ik.Kind()) +} + +// Decode decodes an InternalKey into a keyspan.Span, if it is a range key. If +// keysDst is provided, keys will be appended to keysDst to reduce allocations. +func Decode(ik sstable.InternalKey, val []byte, keysDst []keyspan.Key) (Span, error) { + return rangekey.Decode(ik, val, keysDst) +} diff --git a/vendor/github.com/cockroachdb/pebble/read_state.go b/vendor/github.com/cockroachdb/pebble/read_state.go index a389817..d3a78ba 100644 --- a/vendor/github.com/cockroachdb/pebble/read_state.go +++ b/vendor/github.com/cockroachdb/pebble/read_state.go @@ -20,14 +20,14 @@ import "sync/atomic" // in Go. type readState struct { db *DB - refcnt int32 + refcnt atomic.Int32 current *version memtables flushableList } // ref adds a reference to the readState. func (s *readState) ref() { - atomic.AddInt32(&s.refcnt, 1) + s.refcnt.Add(1) } // unref removes a reference to the readState. If this was the last reference, @@ -35,7 +35,7 @@ func (s *readState) ref() { // is NOT held as version.unref() will acquire it. See unrefLocked() if DB.mu // is held by the caller. func (s *readState) unref() { - if atomic.AddInt32(&s.refcnt, -1) != 0 { + if s.refcnt.Add(-1) != 0 { return } s.current.Unref() @@ -50,10 +50,11 @@ func (s *readState) unref() { // unrefLocked removes a reference to the readState. If this was the last // reference, the reference the readState holds on the version is -// released. Requires DB.mu is held as version.unrefLocked() requires it. See -// unref() if DB.mu is NOT held by the caller. +// released. +// +// DB.mu must be held. See unref() if DB.mu is NOT held by the caller. func (s *readState) unrefLocked() { - if atomic.AddInt32(&s.refcnt, -1) != 0 { + if s.refcnt.Add(-1) != 0 { return } s.current.UnrefLocked() @@ -61,9 +62,8 @@ func (s *readState) unrefLocked() { mem.readerUnrefLocked(true) } - // NB: Unlike readState.unref(), we don't attempt to cleanup newly obsolete - // tables as unrefLocked() is only called during DB shutdown to release the - // current readState. + // In this code path, the caller is responsible for scheduling obsolete table + // deletion as necessary. } // loadReadState returns the current readState. The returned readState must be @@ -82,10 +82,10 @@ func (d *DB) loadReadState() *readState { func (d *DB) updateReadStateLocked(checker func(*DB) error) { s := &readState{ db: d, - refcnt: 1, current: d.mu.versions.currentVersion(), memtables: d.mu.mem.queue, } + s.refcnt.Store(1) s.current.Ref() for _, mem := range s.memtables { mem.readerRef() diff --git a/vendor/github.com/cockroachdb/pebble/record/log_writer.go b/vendor/github.com/cockroachdb/pebble/record/log_writer.go index 9d7ddfa..4aa5d2d 100644 --- a/vendor/github.com/cockroachdb/pebble/record/log_writer.go +++ b/vendor/github.com/cockroachdb/pebble/record/log_writer.go @@ -20,10 +20,11 @@ import ( ) var walSyncLabels = pprof.Labels("pebble", "wal-sync") +var errClosedWriter = errors.New("pebble/record: closed LogWriter") type block struct { // buf[:written] has already been filled with fragments. Updated atomically. - written int32 + written atomic.Int32 // buf[:flushed] has already been flushed to w. flushed int32 buf [blockSize]byte @@ -69,7 +70,7 @@ type syncQueue struct { // // The head index is stored in the most-significant bits so that we can // atomically add to it and the overflow is harmless. - headTail uint64 + headTail atomic.Uint64 // slots is a ring buffer of values stored in this queue. The size must be a // power of 2. A slot is in use until the tail index has moved beyond it. @@ -78,7 +79,7 @@ type syncQueue struct { // blocked is an atomic boolean which indicates whether syncing is currently // blocked or can proceed. It is used by the implementation of // min-sync-interval to block syncing until the min interval has passed. - blocked uint32 + blocked atomic.Bool } const dequeueBits = 32 @@ -91,7 +92,7 @@ func (q *syncQueue) unpack(ptrs uint64) (head, tail uint32) { } func (q *syncQueue) push(wg *sync.WaitGroup, err *error) { - ptrs := atomic.LoadUint64(&q.headTail) + ptrs := q.headTail.Load() head, tail := q.unpack(ptrs) if (tail+uint32(len(q.slots)))&(1< 0 || written > w.block.flushed || !f.syncQ.empty() { break } @@ -442,8 +439,7 @@ func (w *LogWriter) flushLoop(context.Context) { // Found work to do, so no longer idle. workStartTime := time.Now() idleDuration := workStartTime.Sub(idleStartTime) - pending = pending[:len(f.pending)] - copy(pending, f.pending) + pending = append(pending[:0], f.pending...) f.pending = f.pending[:0] f.metrics.PendingBufferLen.AddSample(int64(len(pending))) @@ -459,7 +455,7 @@ func (w *LogWriter) flushLoop(context.Context) { // be ordered after we get the list of sync waiters from syncQ in order to // prevent a race where a waiter adds itself to syncQ, but this thread // picks up the entry in syncQ and not the buffered data. - written := atomic.LoadInt32(&w.block.written) + written := w.block.written.Load() data := w.block.buf[w.block.flushed:written] w.block.flushed = written @@ -562,32 +558,22 @@ func (w *LogWriter) flushBlock(b *block) error { if _, err := w.w.Write(b.buf[b.flushed:]); err != nil { return err } - b.written = 0 + b.written.Store(0) b.flushed = 0 w.free.Lock() w.free.blocks = append(w.free.blocks, b) - w.free.cond.Signal() w.free.Unlock() return nil } // queueBlock queues the current block for writing to the underlying writer, // allocates a new block and reserves space for the next header. -func (w *LogWriter) queueBlock() (waitDuration time.Duration) { +func (w *LogWriter) queueBlock() { // Allocate a new block, blocking until one is available. We do this first // because w.block is protected by w.flusher.Mutex. w.free.Lock() if len(w.free.blocks) == 0 { - if w.free.allocated < cap(w.free.blocks) { - w.free.allocated++ - w.free.blocks = append(w.free.blocks, &block{}) - } else { - now := time.Now() - for len(w.free.blocks) == 0 { - w.free.cond.Wait() - } - waitDuration = time.Since(now) - } + w.free.blocks = append(w.free.blocks, blockPool.Get().(*block)) } nextBlock := w.free.blocks[len(w.free.blocks)-1] w.free.blocks = w.free.blocks[:len(w.free.blocks)-1] @@ -602,28 +588,6 @@ func (w *LogWriter) queueBlock() (waitDuration time.Duration) { f.Unlock() w.blockNum++ - return waitDuration -} - -// ReserveAllFreeBlocksForTesting is used to only for testing. -func (w *LogWriter) ReserveAllFreeBlocksForTesting() (releaseFunc func()) { - w.free.Lock() - defer w.free.Unlock() - free := w.free.blocks - w.free.blocks = nil - return func() { - w.free.Lock() - defer w.free.Unlock() - // It is possible that someone has pushed a free block and w.free.blocks - // is no longer nil. That is harmless. Also, the waiter loops on the - // condition len(w.free.blocks) == 0, so to actually unblock it we need to - // give it a free block. - if len(free) == 0 { - free = append(free, &block{}) - } - w.free.blocks = free - w.free.cond.Broadcast() - } } // Close flushes and syncs any unwritten data and closes the writer. @@ -658,6 +622,7 @@ func (w *LogWriter) Close() error { if f.fsyncLatency != nil { f.fsyncLatency.Observe(float64(syncLatency)) } + free := w.free.blocks f.Unlock() if w.c != nil { @@ -667,7 +632,14 @@ func (w *LogWriter) Close() error { return cerr } } - w.err = errors.New("pebble/record: closed LogWriter") + + for _, b := range free { + b.flushed = 0 + b.written.Store(0) + blockPool.Put(b) + } + + w.err = errClosedWriter return err } @@ -675,20 +647,20 @@ func (w *LogWriter) Close() error { // of the record. // External synchronisation provided by commitPipeline.mu. func (w *LogWriter) WriteRecord(p []byte) (int64, error) { - logSize, _, err := w.SyncRecord(p, nil, nil) + logSize, err := w.SyncRecord(p, nil, nil) return logSize, err } -// SyncRecord writes a complete record. If wg!= nil the record will be +// SyncRecord writes a complete record. If wg != nil the record will be // asynchronously persisted to the underlying writer and done will be called on // the wait group upon completion. Returns the offset just past the end of the // record. // External synchronisation provided by commitPipeline.mu. func (w *LogWriter) SyncRecord( p []byte, wg *sync.WaitGroup, err *error, -) (logSize int64, waitDuration time.Duration, err2 error) { +) (logSize int64, err2 error) { if w.err != nil { - return -1, 0, w.err + return -1, w.err } // The `i == 0` condition ensures we handle empty records. Such records can @@ -696,9 +668,7 @@ func (w *LogWriter) SyncRecord( // MANIFEST is currently written using Writer, it is good to support the same // semantics with LogWriter. for i := 0; i == 0 || len(p) > 0; i++ { - var wd time.Duration - p, wd = w.emitFragment(i, p) - waitDuration += wd + p = w.emitFragment(i, p) } if wg != nil { @@ -712,35 +682,35 @@ func (w *LogWriter) SyncRecord( f.ready.Signal() } - offset := w.blockNum*blockSize + int64(w.block.written) + offset := w.blockNum*blockSize + int64(w.block.written.Load()) // Note that we don't return w.err here as a concurrent call to Close would // race with our read. That's ok because the only error we could be seeing is // one to syncing for which the caller can receive notification of by passing // in a non-nil err argument. - return offset, waitDuration, nil + return offset, nil } // Size returns the current size of the file. // External synchronisation provided by commitPipeline.mu. func (w *LogWriter) Size() int64 { - return w.blockNum*blockSize + int64(w.block.written) + return w.blockNum*blockSize + int64(w.block.written.Load()) } func (w *LogWriter) emitEOFTrailer() { // Write a recyclable chunk header with a different log number. Readers // will treat the header as EOF when the log number does not match. b := w.block - i := b.written + i := b.written.Load() binary.LittleEndian.PutUint32(b.buf[i+0:i+4], 0) // CRC binary.LittleEndian.PutUint16(b.buf[i+4:i+6], 0) // Size b.buf[i+6] = recyclableFullChunkType binary.LittleEndian.PutUint32(b.buf[i+7:i+11], w.logNum+1) // Log number - atomic.StoreInt32(&b.written, i+int32(recyclableHeaderSize)) + b.written.Store(i + int32(recyclableHeaderSize)) } -func (w *LogWriter) emitFragment(n int, p []byte) (remainingP []byte, waitDuration time.Duration) { +func (w *LogWriter) emitFragment(n int, p []byte) (remainingP []byte) { b := w.block - i := b.written + i := b.written.Load() first := n == 0 last := blockSize-i-recyclableHeaderSize >= int32(len(p)) @@ -764,17 +734,17 @@ func (w *LogWriter) emitFragment(n int, p []byte) (remainingP []byte, waitDurati j := i + int32(recyclableHeaderSize+r) binary.LittleEndian.PutUint32(b.buf[i+0:i+4], crc.New(b.buf[i+6:j]).Value()) binary.LittleEndian.PutUint16(b.buf[i+4:i+6], uint16(r)) - atomic.StoreInt32(&b.written, j) + b.written.Store(j) - if blockSize-b.written < recyclableHeaderSize { + if blockSize-b.written.Load() < recyclableHeaderSize { // There is no room for another fragment in the block, so fill the // remaining bytes with zeros and queue the block for flushing. - for i := b.written; i < blockSize; i++ { + for i := b.written.Load(); i < blockSize; i++ { b.buf[i] = 0 } - waitDuration = w.queueBlock() + w.queueBlock() } - return p[r:], waitDuration + return p[r:] } // Metrics must be called after Close. The callee will no longer modify the diff --git a/vendor/github.com/cockroachdb/pebble/scan_internal.go b/vendor/github.com/cockroachdb/pebble/scan_internal.go new file mode 100644 index 0000000..08bebfc --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/scan_internal.go @@ -0,0 +1,1007 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package pebble + +import ( + "context" + "fmt" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/internal/keyspan" + "github.com/cockroachdb/pebble/internal/manifest" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/remote" +) + +const ( + // In skip-shared iteration mode, keys in levels sharedLevelsStart and greater + // (i.e. lower in the LSM) are skipped. + sharedLevelsStart = remote.SharedLevelsStart +) + +// ErrInvalidSkipSharedIteration is returned by ScanInternal if it was called +// with a shared file visitor function, and a file in a shareable level (i.e. +// level >= sharedLevelsStart) was found to not be in shared storage according +// to objstorage.Provider, or not shareable for another reason such as for +// containing keys newer than the snapshot sequence number. +var ErrInvalidSkipSharedIteration = errors.New("pebble: cannot use skip-shared iteration due to non-shareable files in lower levels") + +// SharedSSTMeta represents an sstable on shared storage that can be ingested +// by another pebble instance. This struct must contain all fields that are +// required for a Pebble instance to ingest a foreign sstable on shared storage, +// including constructing any relevant objstorage.Provider / remoteobjcat.Catalog +// data structures, as well as creating virtual FileMetadatas. +// +// Note that the Pebble instance creating and returning a SharedSSTMeta might +// not be the one that created the underlying sstable on shared storage to begin +// with; it's possible for a Pebble instance to reshare an sstable that was +// shared to it. +type SharedSSTMeta struct { + // Backing is the shared object underlying this SST. Can be attached to an + // objstorage.Provider. + Backing objstorage.RemoteObjectBackingHandle + + // Smallest and Largest internal keys for the overall bounds. The kind and + // SeqNum of these will reflect what is physically present on the source Pebble + // instance's view of the sstable; it's up to the ingesting instance to set the + // sequence number in the trailer to match the read-time sequence numbers + // reserved for the level this SST is being ingested into. The Kind is expected + // to remain unchanged by the ingesting instance. + // + // Note that these bounds could be narrower than the bounds of the underlying + // sstable; ScanInternal is expected to truncate sstable bounds to the user key + // bounds passed into that method. + Smallest, Largest InternalKey + + // SmallestRangeKey and LargestRangeKey are internal keys that denote the + // range key bounds of this sstable. Must lie within [Smallest, Largest]. + SmallestRangeKey, LargestRangeKey InternalKey + + // SmallestPointKey and LargestPointKey are internal keys that denote the + // point key bounds of this sstable. Must lie within [Smallest, Largest]. + SmallestPointKey, LargestPointKey InternalKey + + // Level denotes the level at which this file was present at read time. + // For files visited by ScanInternal, this value will only be 5 or 6. + Level uint8 + + // Size contains an estimate of the size of this sstable. + Size uint64 + + // fileNum at time of creation in the creator instance. Only used for + // debugging/tests. + fileNum base.FileNum +} + +func (s *SharedSSTMeta) cloneFromFileMeta(f *fileMetadata) { + *s = SharedSSTMeta{ + Smallest: f.Smallest.Clone(), + Largest: f.Largest.Clone(), + SmallestRangeKey: f.SmallestRangeKey.Clone(), + LargestRangeKey: f.LargestRangeKey.Clone(), + SmallestPointKey: f.SmallestPointKey.Clone(), + LargestPointKey: f.LargestPointKey.Clone(), + Size: f.Size, + fileNum: f.FileNum, + } +} + +type sharedByLevel []SharedSSTMeta + +func (s sharedByLevel) Len() int { return len(s) } +func (s sharedByLevel) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s sharedByLevel) Less(i, j int) bool { return s[i].Level < s[j].Level } + +type pcIterPos int + +const ( + pcIterPosCur pcIterPos = iota + pcIterPosNext +) + +// pointCollapsingIterator is an internalIterator that collapses point keys and +// returns at most one point internal key for each user key. Merges and +// SingleDels are not supported and result in a panic if encountered. Point keys +// deleted by rangedels are considered shadowed and not exposed. +// +// Only used in ScanInternal to return at most one internal key per user key. +type pointCollapsingIterator struct { + iter keyspan.InterleavingIter + pos pcIterPos + comparer *base.Comparer + merge base.Merge + err error + seqNum uint64 + // The current position of `iter`. Always owned by the underlying iter. + iterKey *InternalKey + // The last saved key. findNextEntry and similar methods are expected to save + // the current value of iterKey to savedKey if they're iterating away from the + // current key but still need to retain it. See comments in findNextEntry on + // how this field is used. + // + // At the end of a positioning call: + // - if pos == pcIterPosNext, iterKey is pointing to the next user key owned + // by `iter` while savedKey is holding a copy to our current key. + // - If pos == pcIterPosCur, iterKey is pointing to an `iter`-owned current + // key, and savedKey is either undefined or pointing to a version of the + // current key owned by this iterator (i.e. backed by savedKeyBuf). + savedKey InternalKey + savedKeyBuf []byte + // Value at the current iterator position, at iterKey. + iterValue base.LazyValue + // If fixedSeqNum is non-zero, all emitted points are verified to have this + // fixed sequence number. + fixedSeqNum uint64 +} + +func (p *pointCollapsingIterator) Span() *keyspan.Span { + return p.iter.Span() +} + +// SeekPrefixGE implements the InternalIterator interface. +func (p *pointCollapsingIterator) SeekPrefixGE( + prefix, key []byte, flags base.SeekGEFlags, +) (*base.InternalKey, base.LazyValue) { + p.resetKey() + p.iterKey, p.iterValue = p.iter.SeekPrefixGE(prefix, key, flags) + p.pos = pcIterPosCur + if p.iterKey == nil { + return nil, base.LazyValue{} + } + return p.findNextEntry() +} + +// SeekGE implements the InternalIterator interface. +func (p *pointCollapsingIterator) SeekGE( + key []byte, flags base.SeekGEFlags, +) (*base.InternalKey, base.LazyValue) { + p.resetKey() + p.iterKey, p.iterValue = p.iter.SeekGE(key, flags) + p.pos = pcIterPosCur + if p.iterKey == nil { + return nil, base.LazyValue{} + } + return p.findNextEntry() +} + +// SeekLT implements the InternalIterator interface. +func (p *pointCollapsingIterator) SeekLT( + key []byte, flags base.SeekLTFlags, +) (*base.InternalKey, base.LazyValue) { + panic("unimplemented") +} + +func (p *pointCollapsingIterator) resetKey() { + p.savedKey.UserKey = p.savedKeyBuf[:0] + p.savedKey.Trailer = 0 + p.iterKey = nil + p.pos = pcIterPosCur +} + +func (p *pointCollapsingIterator) verifySeqNum(key *base.InternalKey) *base.InternalKey { + if !invariants.Enabled { + return key + } + if p.fixedSeqNum == 0 || key == nil || key.Kind() == InternalKeyKindRangeDelete { + return key + } + if key.SeqNum() != p.fixedSeqNum { + panic(fmt.Sprintf("expected foreign point key to have seqnum %d, got %d", p.fixedSeqNum, key.SeqNum())) + } + return key +} + +// findNextEntry is called to return the next key. p.iter must be positioned at the +// start of the first user key we are interested in. +func (p *pointCollapsingIterator) findNextEntry() (*base.InternalKey, base.LazyValue) { + p.saveKey() + // Saves a comparison in the fast path + firstIteration := true + for p.iterKey != nil { + // NB: p.savedKey is either the current key (iff p.iterKey == firstKey), + // or the previous key. + if !firstIteration && !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) { + p.saveKey() + continue + } + firstIteration = false + if s := p.iter.Span(); s != nil && s.CoversAt(p.seqNum, p.iterKey.SeqNum()) { + // All future keys for this user key must be deleted. + if p.savedKey.Kind() == InternalKeyKindSingleDelete { + panic("cannot process singledel key in point collapsing iterator") + } + // Fast forward to the next user key. + p.saveKey() + p.iterKey, p.iterValue = p.iter.Next() + for p.iterKey != nil && p.savedKey.SeqNum() >= p.iterKey.SeqNum() && p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) { + p.iterKey, p.iterValue = p.iter.Next() + } + continue + } + switch p.savedKey.Kind() { + case InternalKeyKindSet, InternalKeyKindDelete, InternalKeyKindSetWithDelete, InternalKeyKindDeleteSized: + // Note that we return SETs directly, even if they would otherwise get + // compacted into a Del to turn into a SetWithDelete. This is a fast + // path optimization that can break SINGLEDEL determinism. To lead to + // consistent SINGLEDEL behaviour, this iterator should *not* be used for + // a keyspace where SINGLEDELs could be in use. If this iterator observes + // a SINGLEDEL as the first internal key for a user key, it will panic. + // + // As p.value is a lazy value owned by the child iterator, we can thread + // it through without loading it into p.valueBuf. + // + // TODO(bilal): We can even avoid saving the key in this fast path if + // we are in a block where setHasSamePrefix = false in a v3 sstable, + // guaranteeing that there's only one internal key for each user key. + // Thread this logic through the sstable iterators and/or consider + // collapsing (ha) this logic into the sstable iterators that are aware + // of blocks and can determine user key changes without doing key saves + // or comparisons. + p.pos = pcIterPosCur + return p.verifySeqNum(p.iterKey), p.iterValue + case InternalKeyKindSingleDelete: + // Panic, as this iterator is not expected to observe single deletes. + panic("cannot process singledel key in point collapsing iterator") + case InternalKeyKindMerge: + // Panic, as this iterator is not expected to observe merges. + panic("cannot process merge key in point collapsing iterator") + case InternalKeyKindRangeDelete: + // These are interleaved by the interleaving iterator ahead of all points. + // We should pass them as-is, but also account for any points ahead of + // them. + p.pos = pcIterPosCur + return p.verifySeqNum(p.iterKey), p.iterValue + default: + panic(fmt.Sprintf("unexpected kind: %d", p.iterKey.Kind())) + } + } + p.resetKey() + return nil, base.LazyValue{} +} + +// First implements the InternalIterator interface. +func (p *pointCollapsingIterator) First() (*base.InternalKey, base.LazyValue) { + p.resetKey() + p.iterKey, p.iterValue = p.iter.First() + p.pos = pcIterPosCur + if p.iterKey == nil { + return nil, base.LazyValue{} + } + return p.findNextEntry() +} + +// Last implements the InternalIterator interface. +func (p *pointCollapsingIterator) Last() (*base.InternalKey, base.LazyValue) { + panic("unimplemented") +} + +func (p *pointCollapsingIterator) saveKey() { + if p.iterKey == nil { + p.savedKey = InternalKey{UserKey: p.savedKeyBuf[:0]} + return + } + p.savedKeyBuf = append(p.savedKeyBuf[:0], p.iterKey.UserKey...) + p.savedKey = InternalKey{UserKey: p.savedKeyBuf, Trailer: p.iterKey.Trailer} +} + +// Next implements the InternalIterator interface. +func (p *pointCollapsingIterator) Next() (*base.InternalKey, base.LazyValue) { + switch p.pos { + case pcIterPosCur: + p.saveKey() + if p.iterKey != nil && p.iterKey.Kind() == InternalKeyKindRangeDelete { + // Step over the interleaved range delete and process the very next + // internal key, even if it's at the same user key. This is because a + // point for that user key has not been returned yet. + p.iterKey, p.iterValue = p.iter.Next() + break + } + // Fast forward to the next user key. + key, val := p.iter.Next() + // p.iterKey.SeqNum() >= key.SeqNum() is an optimization that allows us to + // use p.iterKey.SeqNum() < key.SeqNum() as a sign that the user key has + // changed, without needing to do the full key comparison. + for key != nil && p.savedKey.SeqNum() >= key.SeqNum() && + p.comparer.Equal(p.savedKey.UserKey, key.UserKey) { + key, val = p.iter.Next() + } + if key == nil { + // There are no keys to return. + p.resetKey() + return nil, base.LazyValue{} + } + p.iterKey, p.iterValue = key, val + case pcIterPosNext: + p.pos = pcIterPosCur + } + if p.iterKey == nil { + p.resetKey() + return nil, base.LazyValue{} + } + return p.findNextEntry() +} + +// NextPrefix implements the InternalIterator interface. +func (p *pointCollapsingIterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) { + panic("unimplemented") +} + +// Prev implements the InternalIterator interface. +func (p *pointCollapsingIterator) Prev() (*base.InternalKey, base.LazyValue) { + panic("unimplemented") +} + +// Error implements the InternalIterator interface. +func (p *pointCollapsingIterator) Error() error { + if p.err != nil { + return p.err + } + return p.iter.Error() +} + +// Close implements the InternalIterator interface. +func (p *pointCollapsingIterator) Close() error { + return p.iter.Close() +} + +// SetBounds implements the InternalIterator interface. +func (p *pointCollapsingIterator) SetBounds(lower, upper []byte) { + p.resetKey() + p.iter.SetBounds(lower, upper) +} + +// String implements the InternalIterator interface. +func (p *pointCollapsingIterator) String() string { + return p.iter.String() +} + +var _ internalIterator = &pointCollapsingIterator{} + +// IteratorLevelKind is used to denote whether the current ScanInternal iterator +// is unknown, belongs to a flushable, or belongs to an LSM level type. +type IteratorLevelKind int8 + +const ( + // IteratorLevelUnknown indicates an unknown LSM level. + IteratorLevelUnknown IteratorLevelKind = iota + // IteratorLevelLSM indicates an LSM level. + IteratorLevelLSM + // IteratorLevelFlushable indicates a flushable (i.e. memtable). + IteratorLevelFlushable +) + +// IteratorLevel is used with scanInternalIterator to surface additional iterator-specific info where possible. +// Note: this is struct is only provided for point keys. +type IteratorLevel struct { + Kind IteratorLevelKind + // FlushableIndex indicates the position within the flushable queue of this level. + // Only valid if kind == IteratorLevelFlushable. + FlushableIndex int + // The level within the LSM. Only valid if Kind == IteratorLevelLSM. + Level int + // Sublevel is only valid if Kind == IteratorLevelLSM and Level == 0. + Sublevel int +} + +// scanInternalIterator is an iterator that returns all internal keys, including +// tombstones. For instance, an InternalKeyKindDelete would be returned as an +// InternalKeyKindDelete instead of the iterator skipping over to the next key. +// Internal keys within a user key are collapsed, eg. if there are two SETs, the +// one with the higher sequence is returned. Useful if an external user of Pebble +// needs to observe and rebuild Pebble's history of internal keys, such as in +// node-to-node replication. For use with {db,snapshot}.ScanInternal(). +// +// scanInternalIterator is expected to ignore point keys deleted by range +// deletions, and range keys shadowed by a range key unset or delete, however it +// *must* return the range delete as well as the range key unset/delete that did +// the shadowing. +type scanInternalIterator struct { + db *DB + opts scanInternalOptions + comparer *base.Comparer + merge Merge + iter internalIterator + readState *readState + version *version + rangeKey *iteratorRangeKeyState + pointKeyIter internalIterator + iterKey *InternalKey + iterValue LazyValue + alloc *iterAlloc + newIters tableNewIters + newIterRangeKey keyspan.TableNewSpanIter + seqNum uint64 + iterLevels []IteratorLevel + mergingIter *mergingIter + + // boundsBuf holds two buffers used to store the lower and upper bounds. + // Whenever the InternalIterator's bounds change, the new bounds are copied + // into boundsBuf[boundsBufIdx]. The two bounds share a slice to reduce + // allocations. opts.LowerBound and opts.UpperBound point into this slice. + boundsBuf [2][]byte + boundsBufIdx int +} + +// truncateSharedFile truncates a shared file's [Smallest, Largest] fields to +// [lower, upper), potentially opening iterators on the file to find keys within +// the requested bounds. A SharedSSTMeta is produced that is suitable for +// external consumption by other Pebble instances. If shouldSkip is true, this +// file does not contain any keys in [lower, upper) and can be skipped. +// +// TODO(bilal): If opening iterators and doing reads in this method is too +// inefficient, consider producing non-tight file bounds instead. +func (d *DB) truncateSharedFile( + ctx context.Context, + lower, upper []byte, + level int, + file *fileMetadata, + objMeta objstorage.ObjectMetadata, +) (sst *SharedSSTMeta, shouldSkip bool, err error) { + cmp := d.cmp + sst = &SharedSSTMeta{} + sst.cloneFromFileMeta(file) + sst.Level = uint8(level) + sst.Backing, err = d.objProvider.RemoteObjectBacking(&objMeta) + if err != nil { + return nil, false, err + } + needsLowerTruncate := cmp(lower, file.Smallest.UserKey) > 0 + needsUpperTruncate := cmp(upper, file.Largest.UserKey) < 0 || (cmp(upper, file.Largest.UserKey) == 0 && !file.Largest.IsExclusiveSentinel()) + // Fast path: file is entirely within [lower, upper). + if !needsLowerTruncate && !needsUpperTruncate { + return sst, false, nil + } + + // We will need to truncate file bounds in at least one direction. Open all + // relevant iterators. + iter, rangeDelIter, err := d.newIters(ctx, file, &IterOptions{ + LowerBound: lower, + UpperBound: upper, + level: manifest.Level(level), + }, internalIterOpts{}) + if err != nil { + return nil, false, err + } + defer iter.Close() + if rangeDelIter != nil { + rangeDelIter = keyspan.Truncate( + cmp, rangeDelIter, lower, upper, nil, nil, + false, /* panicOnUpperTruncate */ + ) + defer rangeDelIter.Close() + } + rangeKeyIter, err := d.tableNewRangeKeyIter(file, keyspan.SpanIterOptions{}) + if err != nil { + return nil, false, err + } + if rangeKeyIter != nil { + rangeKeyIter = keyspan.Truncate( + cmp, rangeKeyIter, lower, upper, nil, nil, + false, /* panicOnUpperTruncate */ + ) + defer rangeKeyIter.Close() + } + // Check if we need to truncate on the left side. This means finding a new + // LargestPointKey and LargestRangeKey that is >= lower. + if needsLowerTruncate { + sst.SmallestPointKey.UserKey = sst.SmallestPointKey.UserKey[:0] + sst.SmallestPointKey.Trailer = 0 + key, _ := iter.SeekGE(lower, base.SeekGEFlagsNone) + foundPointKey := key != nil + if key != nil { + sst.SmallestPointKey.CopyFrom(*key) + } + if rangeDelIter != nil { + span := rangeDelIter.SeekGE(lower) + if span != nil && (len(sst.SmallestPointKey.UserKey) == 0 || base.InternalCompare(cmp, span.SmallestKey(), sst.SmallestPointKey) < 0) { + sst.SmallestPointKey.CopyFrom(span.SmallestKey()) + foundPointKey = true + } + } + if !foundPointKey { + // There are no point keys in the span we're interested in. + sst.SmallestPointKey = InternalKey{} + sst.LargestPointKey = InternalKey{} + } + sst.SmallestRangeKey.UserKey = sst.SmallestRangeKey.UserKey[:0] + sst.SmallestRangeKey.Trailer = 0 + if rangeKeyIter != nil { + span := rangeKeyIter.SeekGE(lower) + if span != nil { + sst.SmallestRangeKey.CopyFrom(span.SmallestKey()) + } else { + // There are no range keys in the span we're interested in. + sst.SmallestRangeKey = InternalKey{} + sst.LargestRangeKey = InternalKey{} + } + } + } + // Check if we need to truncate on the right side. This means finding a new + // LargestPointKey and LargestRangeKey that is < upper. + if needsUpperTruncate { + sst.LargestPointKey.UserKey = sst.LargestPointKey.UserKey[:0] + sst.LargestPointKey.Trailer = 0 + key, _ := iter.SeekLT(upper, base.SeekLTFlagsNone) + foundPointKey := key != nil + if key != nil { + sst.LargestPointKey.CopyFrom(*key) + } + if rangeDelIter != nil { + span := rangeDelIter.SeekLT(upper) + if span != nil && (len(sst.LargestPointKey.UserKey) == 0 || base.InternalCompare(cmp, span.LargestKey(), sst.LargestPointKey) > 0) { + sst.LargestPointKey.CopyFrom(span.LargestKey()) + foundPointKey = true + } + } + if !foundPointKey { + // There are no point keys in the span we're interested in. + sst.SmallestPointKey = InternalKey{} + sst.LargestPointKey = InternalKey{} + } + sst.LargestRangeKey.UserKey = sst.LargestRangeKey.UserKey[:0] + sst.LargestRangeKey.Trailer = 0 + if rangeKeyIter != nil { + span := rangeKeyIter.SeekLT(upper) + if span != nil { + sst.LargestRangeKey.CopyFrom(span.LargestKey()) + } else { + // There are no range keys in the span we're interested in. + sst.SmallestRangeKey = InternalKey{} + sst.LargestRangeKey = InternalKey{} + } + } + } + // Set overall bounds based on {Smallest,Largest}{Point,Range}Key. + switch { + case len(sst.SmallestRangeKey.UserKey) == 0: + sst.Smallest = sst.SmallestPointKey + case len(sst.SmallestPointKey.UserKey) == 0: + sst.Smallest = sst.SmallestRangeKey + default: + sst.Smallest = sst.SmallestPointKey + if base.InternalCompare(cmp, sst.SmallestRangeKey, sst.SmallestPointKey) < 0 { + sst.Smallest = sst.SmallestRangeKey + } + } + switch { + case len(sst.LargestRangeKey.UserKey) == 0: + sst.Largest = sst.LargestPointKey + case len(sst.LargestPointKey.UserKey) == 0: + sst.Largest = sst.LargestRangeKey + default: + sst.Largest = sst.LargestPointKey + if base.InternalCompare(cmp, sst.LargestRangeKey, sst.LargestPointKey) > 0 { + sst.Largest = sst.LargestRangeKey + } + } + // On rare occasion, a file might overlap with [lower, upper) but not actually + // have any keys within those bounds. Skip such files. + if len(sst.Smallest.UserKey) == 0 { + return nil, true, nil + } + sst.Size, err = d.tableCache.estimateSize(file, sst.Smallest.UserKey, sst.Largest.UserKey) + if err != nil { + return nil, false, err + } + // On occasion, estimateSize gives us a low estimate, i.e. a 0 file size. This + // can cause panics in places where we divide by file sizes. Correct for it + // here. + if sst.Size == 0 { + sst.Size = 1 + } + return sst, false, nil +} + +func scanInternalImpl( + ctx context.Context, lower, upper []byte, iter *scanInternalIterator, opts *scanInternalOptions, +) error { + if opts.visitSharedFile != nil && (lower == nil || upper == nil) { + panic("lower and upper bounds must be specified in skip-shared iteration mode") + } + // Before starting iteration, check if any files in levels sharedLevelsStart + // and below are *not* shared. Error out if that is the case, as skip-shared + // iteration will not produce a consistent point-in-time view of this range + // of keys. For files that are shared, call visitSharedFile with a truncated + // version of that file. + cmp := iter.comparer.Compare + provider := iter.db.ObjProvider() + seqNum := iter.seqNum + current := iter.version + if current == nil { + current = iter.readState.current + } + if opts.visitSharedFile != nil { + if provider == nil { + panic("expected non-nil Provider in skip-shared iteration mode") + } + for level := sharedLevelsStart; level < numLevels; level++ { + files := current.Levels[level].Iter() + for f := files.SeekGE(cmp, lower); f != nil && cmp(f.Smallest.UserKey, upper) < 0; f = files.Next() { + var objMeta objstorage.ObjectMetadata + var err error + objMeta, err = provider.Lookup(fileTypeTable, f.FileBacking.DiskFileNum) + if err != nil { + return err + } + if !objMeta.IsShared() { + return errors.Wrapf(ErrInvalidSkipSharedIteration, "file %s is not shared", objMeta.DiskFileNum) + } + if !base.Visible(f.LargestSeqNum, seqNum, base.InternalKeySeqNumMax) { + return errors.Wrapf(ErrInvalidSkipSharedIteration, "file %s contains keys newer than snapshot", objMeta.DiskFileNum) + } + var sst *SharedSSTMeta + var skip bool + sst, skip, err = iter.db.truncateSharedFile(ctx, lower, upper, level, f, objMeta) + if err != nil { + return err + } + if skip { + continue + } + if err = opts.visitSharedFile(sst); err != nil { + return err + } + } + } + } + + for valid := iter.seekGE(lower); valid && iter.error() == nil; valid = iter.next() { + key := iter.unsafeKey() + + if opts.rateLimitFunc != nil { + if err := opts.rateLimitFunc(key, iter.lazyValue()); err != nil { + return err + } + } + + switch key.Kind() { + case InternalKeyKindRangeKeyDelete, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet: + if opts.visitRangeKey != nil { + span := iter.unsafeSpan() + // NB: The caller isn't interested in the sequence numbers of these + // range keys. Rather, the caller wants them to be in trailer order + // _after_ zeroing of sequence numbers. Copy span.Keys, sort it, and then + // call visitRangeKey. + keysCopy := make([]keyspan.Key, len(span.Keys)) + for i := range span.Keys { + keysCopy[i] = span.Keys[i] + keysCopy[i].Trailer = base.MakeTrailer(0, span.Keys[i].Kind()) + } + keyspan.SortKeysByTrailer(&keysCopy) + if err := opts.visitRangeKey(span.Start, span.End, keysCopy); err != nil { + return err + } + } + case InternalKeyKindRangeDelete: + if opts.visitRangeDel != nil { + rangeDel := iter.unsafeRangeDel() + if err := opts.visitRangeDel(rangeDel.Start, rangeDel.End, rangeDel.LargestSeqNum()); err != nil { + return err + } + } + default: + if opts.visitPointKey != nil { + var info IteratorLevel + if len(iter.mergingIter.heap.items) > 0 { + mergingIterIdx := iter.mergingIter.heap.items[0].index + info = iter.iterLevels[mergingIterIdx] + } else { + info = IteratorLevel{Kind: IteratorLevelUnknown} + } + val := iter.lazyValue() + if err := opts.visitPointKey(key, val, info); err != nil { + return err + } + } + } + } + + return nil +} + +// constructPointIter constructs a merging iterator and sets i.iter to it. +func (i *scanInternalIterator) constructPointIter(memtables flushableList, buf *iterAlloc) { + // Merging levels and levels from iterAlloc. + mlevels := buf.mlevels[:0] + levels := buf.levels[:0] + + // We compute the number of levels needed ahead of time and reallocate a slice if + // the array from the iterAlloc isn't large enough. Doing this allocation once + // should improve the performance. + numMergingLevels := len(memtables) + numLevelIters := 0 + + current := i.version + if current == nil { + current = i.readState.current + } + numMergingLevels += len(current.L0SublevelFiles) + numLevelIters += len(current.L0SublevelFiles) + + for level := 1; level < len(current.Levels); level++ { + if current.Levels[level].Empty() { + continue + } + if i.opts.skipSharedLevels && level >= sharedLevelsStart { + continue + } + numMergingLevels++ + numLevelIters++ + } + + if numMergingLevels > cap(mlevels) { + mlevels = make([]mergingIterLevel, 0, numMergingLevels) + } + if numLevelIters > cap(levels) { + levels = make([]levelIter, 0, numLevelIters) + } + // TODO(bilal): Push these into the iterAlloc buf. + var rangeDelMiter keyspan.MergingIter + rangeDelIters := make([]keyspan.FragmentIterator, 0, numMergingLevels) + rangeDelLevels := make([]keyspan.LevelIter, 0, numLevelIters) + + i.iterLevels = make([]IteratorLevel, numMergingLevels) + mlevelsIndex := 0 + + // Next are the memtables. + for j := len(memtables) - 1; j >= 0; j-- { + mem := memtables[j] + mlevels = append(mlevels, mergingIterLevel{ + iter: mem.newIter(&i.opts.IterOptions), + }) + i.iterLevels[mlevelsIndex] = IteratorLevel{ + Kind: IteratorLevelFlushable, + FlushableIndex: j, + } + mlevelsIndex++ + if rdi := mem.newRangeDelIter(&i.opts.IterOptions); rdi != nil { + rangeDelIters = append(rangeDelIters, rdi) + } + } + + // Next are the file levels: L0 sub-levels followed by lower levels. + levelsIndex := len(levels) + mlevels = mlevels[:numMergingLevels] + levels = levels[:numLevelIters] + rangeDelLevels = rangeDelLevels[:numLevelIters] + i.opts.IterOptions.snapshotForHideObsoletePoints = i.seqNum + addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Level) { + li := &levels[levelsIndex] + rli := &rangeDelLevels[levelsIndex] + + li.init( + context.Background(), i.opts.IterOptions, i.comparer, i.newIters, files, level, + internalIterOpts{}) + li.initBoundaryContext(&mlevels[mlevelsIndex].levelIterBoundaryContext) + mlevels[mlevelsIndex].iter = li + rli.Init(keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}, + i.comparer.Compare, tableNewRangeDelIter(context.Background(), i.newIters), files, level, + manifest.KeyTypePoint) + rangeDelIters = append(rangeDelIters, rli) + + levelsIndex++ + mlevelsIndex++ + } + + for j := len(current.L0SublevelFiles) - 1; j >= 0; j-- { + i.iterLevels[mlevelsIndex] = IteratorLevel{ + Kind: IteratorLevelLSM, + Level: 0, + Sublevel: j, + } + addLevelIterForFiles(current.L0SublevelFiles[j].Iter(), manifest.L0Sublevel(j)) + } + // Add level iterators for the non-empty non-L0 levels. + for level := 1; level < numLevels; level++ { + if current.Levels[level].Empty() { + continue + } + if i.opts.skipSharedLevels && level >= sharedLevelsStart { + continue + } + i.iterLevels[mlevelsIndex] = IteratorLevel{Kind: IteratorLevelLSM, Level: level} + addLevelIterForFiles(current.Levels[level].Iter(), manifest.Level(level)) + } + + buf.merging.init(&i.opts.IterOptions, &InternalIteratorStats{}, i.comparer.Compare, i.comparer.Split, mlevels...) + buf.merging.snapshot = i.seqNum + rangeDelMiter.Init(i.comparer.Compare, keyspan.VisibleTransform(i.seqNum), new(keyspan.MergingBuffers), rangeDelIters...) + + if i.opts.includeObsoleteKeys { + iiter := &keyspan.InterleavingIter{} + iiter.Init(i.comparer, &buf.merging, &rangeDelMiter, + keyspan.InterleavingIterOpts{ + LowerBound: i.opts.LowerBound, + UpperBound: i.opts.UpperBound, + }) + i.pointKeyIter = iiter + } else { + pcIter := &pointCollapsingIterator{ + comparer: i.comparer, + merge: i.merge, + seqNum: i.seqNum, + } + pcIter.iter.Init(i.comparer, &buf.merging, &rangeDelMiter, keyspan.InterleavingIterOpts{ + LowerBound: i.opts.LowerBound, + UpperBound: i.opts.UpperBound, + }) + i.pointKeyIter = pcIter + } + i.iter = i.pointKeyIter +} + +// constructRangeKeyIter constructs the range-key iterator stack, populating +// i.rangeKey.rangeKeyIter with the resulting iterator. This is similar to +// Iterator.constructRangeKeyIter, except it doesn't handle batches and ensures +// iterConfig does *not* elide unsets/deletes. +func (i *scanInternalIterator) constructRangeKeyIter() { + // We want the bounded iter from iterConfig, but not the collapsing of + // RangeKeyUnsets and RangeKeyDels. + i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init( + i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound, + nil /* hasPrefix */, nil /* prefix */, true, /* internalKeys */ + &i.rangeKey.rangeKeyBuffers.internal) + + // Next are the flushables: memtables and large batches. + if i.readState != nil { + for j := len(i.readState.memtables) - 1; j >= 0; j-- { + mem := i.readState.memtables[j] + // We only need to read from memtables which contain sequence numbers older + // than seqNum. + if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum { + continue + } + if rki := mem.newRangeKeyIter(&i.opts.IterOptions); rki != nil { + i.rangeKey.iterConfig.AddLevel(rki) + } + } + } + + current := i.version + if current == nil { + current = i.readState.current + } + // Next are the file levels: L0 sub-levels followed by lower levels. + // + // Add file-specific iterators for L0 files containing range keys. This is less + // efficient than using levelIters for sublevels of L0 files containing + // range keys, but range keys are expected to be sparse anyway, reducing the + // cost benefit of maintaining a separate L0Sublevels instance for range key + // files and then using it here. + // + // NB: We iterate L0's files in reverse order. They're sorted by + // LargestSeqNum ascending, and we need to add them to the merging iterator + // in LargestSeqNum descending to preserve the merging iterator's invariants + // around Key Trailer order. + iter := current.RangeKeyLevels[0].Iter() + for f := iter.Last(); f != nil; f = iter.Prev() { + spanIter, err := i.newIterRangeKey(f, i.opts.SpanIterOptions()) + if err != nil { + i.rangeKey.iterConfig.AddLevel(&errorKeyspanIter{err: err}) + continue + } + i.rangeKey.iterConfig.AddLevel(spanIter) + } + + // Add level iterators for the non-empty non-L0 levels. + for level := 1; level < len(current.RangeKeyLevels); level++ { + if current.RangeKeyLevels[level].Empty() { + continue + } + if i.opts.skipSharedLevels && level >= sharedLevelsStart { + continue + } + li := i.rangeKey.iterConfig.NewLevelIter() + spanIterOpts := i.opts.SpanIterOptions() + li.Init(spanIterOpts, i.comparer.Compare, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(), + manifest.Level(level), manifest.KeyTypeRange) + i.rangeKey.iterConfig.AddLevel(li) + } +} + +// seekGE seeks this iterator to the first key that's greater than or equal +// to the specified user key. +func (i *scanInternalIterator) seekGE(key []byte) bool { + i.iterKey, i.iterValue = i.iter.SeekGE(key, base.SeekGEFlagsNone) + return i.iterKey != nil +} + +// unsafeKey returns the unsafe InternalKey at the current position. The value +// is nil if the iterator is invalid or exhausted. +func (i *scanInternalIterator) unsafeKey() *InternalKey { + return i.iterKey +} + +// lazyValue returns a value pointer to the value at the current iterator +// position. Behaviour undefined if unsafeKey() returns a Range key or Rangedel +// kind key. +func (i *scanInternalIterator) lazyValue() LazyValue { + return i.iterValue +} + +// unsafeRangeDel returns a range key span. Behaviour undefined if UnsafeKey returns +// a non-rangedel kind. +func (i *scanInternalIterator) unsafeRangeDel() *keyspan.Span { + type spanInternalIterator interface { + Span() *keyspan.Span + } + return i.pointKeyIter.(spanInternalIterator).Span() +} + +// unsafeSpan returns a range key span. Behaviour undefined if UnsafeKey returns +// a non-rangekey type. +func (i *scanInternalIterator) unsafeSpan() *keyspan.Span { + return i.rangeKey.iiter.Span() +} + +// next advances the iterator in the forward direction, and returns the +// iterator's new validity state. +func (i *scanInternalIterator) next() bool { + i.iterKey, i.iterValue = i.iter.Next() + return i.iterKey != nil +} + +// error returns an error from the internal iterator, if there's any. +func (i *scanInternalIterator) error() error { + return i.iter.Error() +} + +// close closes this iterator, and releases any pooled objects. +func (i *scanInternalIterator) close() error { + if err := i.iter.Close(); err != nil { + return err + } + if i.readState != nil { + i.readState.unref() + } + if i.version != nil { + i.version.Unref() + } + if i.rangeKey != nil { + i.rangeKey.PrepareForReuse() + *i.rangeKey = iteratorRangeKeyState{ + rangeKeyBuffers: i.rangeKey.rangeKeyBuffers, + } + iterRangeKeyStateAllocPool.Put(i.rangeKey) + i.rangeKey = nil + } + if alloc := i.alloc; alloc != nil { + for j := range i.boundsBuf { + if cap(i.boundsBuf[j]) >= maxKeyBufCacheSize { + alloc.boundsBuf[j] = nil + } else { + alloc.boundsBuf[j] = i.boundsBuf[j] + } + } + *alloc = iterAlloc{ + keyBuf: alloc.keyBuf[:0], + boundsBuf: alloc.boundsBuf, + prefixOrFullSeekKey: alloc.prefixOrFullSeekKey[:0], + } + iterAllocPool.Put(alloc) + i.alloc = nil + } + return nil +} + +func (i *scanInternalIterator) initializeBoundBufs(lower, upper []byte) { + buf := i.boundsBuf[i.boundsBufIdx][:0] + if lower != nil { + buf = append(buf, lower...) + i.opts.LowerBound = buf + } else { + i.opts.LowerBound = nil + } + if upper != nil { + buf = append(buf, upper...) + i.opts.UpperBound = buf[len(buf)-len(upper):] + } else { + i.opts.UpperBound = nil + } + i.boundsBuf[i.boundsBufIdx] = buf + i.boundsBufIdx = 1 - i.boundsBufIdx +} diff --git a/vendor/github.com/cockroachdb/pebble/snapshot.go b/vendor/github.com/cockroachdb/pebble/snapshot.go index afd5bda..d745e8a 100644 --- a/vendor/github.com/cockroachdb/pebble/snapshot.go +++ b/vendor/github.com/cockroachdb/pebble/snapshot.go @@ -8,16 +8,29 @@ import ( "context" "io" "math" + "sync" + "sync/atomic" + "time" - "github.com/cockroachdb/pebble/internal/keyspan" + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/rangekey" ) +// ErrSnapshotExcised is returned from WaitForFileOnlySnapshot if an excise +// overlapping with one of the EventuallyFileOnlySnapshot's KeyRanges gets +// applied before the transition of that EFOS to a file-only snapshot. +var ErrSnapshotExcised = errors.New("pebble: snapshot excised before conversion to file-only snapshot") + // Snapshot provides a read-only point-in-time view of the DB state. type Snapshot struct { // The db the snapshot was created from. db *DB seqNum uint64 + // Set if part of an EventuallyFileOnlySnapshot. + efos *EventuallyFileOnlySnapshot + // The list the snapshot is linked into. list *snapshotList @@ -44,17 +57,17 @@ func (s *Snapshot) Get(key []byte) ([]byte, io.Closer, error) { // NewIter returns an iterator that is unpositioned (Iterator.Valid() will // return false). The iterator can be positioned via a call to SeekGE, // SeekLT, First or Last. -func (s *Snapshot) NewIter(o *IterOptions) *Iterator { +func (s *Snapshot) NewIter(o *IterOptions) (*Iterator, error) { return s.NewIterWithContext(context.Background(), o) } // NewIterWithContext is like NewIter, and additionally accepts a context for // tracing. -func (s *Snapshot) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterator { +func (s *Snapshot) NewIterWithContext(ctx context.Context, o *IterOptions) (*Iterator, error) { if s.db == nil { panic(ErrClosed) } - return s.db.newIter(ctx, nil /* batch */, s, o) + return s.db.newIter(ctx, nil /* batch */, snapshotIterOpts{seqNum: s.seqNum}, o), nil } // ScanInternal scans all internal keys within the specified bounds, truncating @@ -64,33 +77,38 @@ func (s *Snapshot) NewIterWithContext(ctx context.Context, o *IterOptions) *Iter // See comment on db.ScanInternal for the behaviour that can be expected of // point keys deleted by range dels and keys masked by range keys. func (s *Snapshot) ScanInternal( + ctx context.Context, lower, upper []byte, - visitPointKey func(key *InternalKey, value LazyValue) error, + visitPointKey func(key *InternalKey, value LazyValue, iterInfo IteratorLevel) error, visitRangeDel func(start, end []byte, seqNum uint64) error, - visitRangeKey func(start, end []byte, keys []keyspan.Key) error, + visitRangeKey func(start, end []byte, keys []rangekey.Key) error, + visitSharedFile func(sst *SharedSSTMeta) error, ) error { if s.db == nil { panic(ErrClosed) } - iter := s.db.newInternalIter(s, &IterOptions{ - KeyTypes: IterKeyTypePointsAndRanges, - LowerBound: lower, - UpperBound: upper, - }) + scanInternalOpts := &scanInternalOptions{ + visitPointKey: visitPointKey, + visitRangeDel: visitRangeDel, + visitRangeKey: visitRangeKey, + visitSharedFile: visitSharedFile, + skipSharedLevels: visitSharedFile != nil, + IterOptions: IterOptions{ + KeyTypes: IterKeyTypePointsAndRanges, + LowerBound: lower, + UpperBound: upper, + }, + } + + iter := s.db.newInternalIter(snapshotIterOpts{seqNum: s.seqNum}, scanInternalOpts) defer iter.close() - return scanInternalImpl(lower, iter, visitPointKey, visitRangeDel, visitRangeKey) + return scanInternalImpl(ctx, lower, upper, iter, scanInternalOpts) } -// Close closes the snapshot, releasing its resources. Close must be called. -// Failure to do so will result in a tiny memory leak and a large leak of -// resources on disk due to the entries the snapshot is preventing from being -// deleted. -func (s *Snapshot) Close() error { - if s.db == nil { - panic(ErrClosed) - } - s.db.mu.Lock() +// closeLocked is similar to Close(), except it requires that db.mu be held +// by the caller. +func (s *Snapshot) closeLocked() error { s.db.mu.snapshots.remove(s) // If s was the previous earliest snapshot, we might be able to reclaim @@ -98,11 +116,26 @@ func (s *Snapshot) Close() error { if e := s.db.mu.snapshots.earliest(); e > s.seqNum { s.db.maybeScheduleCompactionPicker(pickElisionOnly) } - s.db.mu.Unlock() s.db = nil return nil } +// Close closes the snapshot, releasing its resources. Close must be called. +// Failure to do so will result in a tiny memory leak and a large leak of +// resources on disk due to the entries the snapshot is preventing from being +// deleted. +// +// d.mu must NOT be held by the caller. +func (s *Snapshot) Close() error { + db := s.db + if db == nil { + panic(ErrClosed) + } + db.mu.Lock() + defer db.mu.Unlock() + return s.closeLocked() +} + type snapshotList struct { root Snapshot } @@ -170,3 +203,345 @@ func (l *snapshotList) remove(s *Snapshot) { s.prev = nil // avoid memory leaks s.list = nil // avoid memory leaks } + +// EventuallyFileOnlySnapshot (aka EFOS) provides a read-only point-in-time view +// of the database state, similar to Snapshot. An EventuallyFileOnlySnapshot +// induces less write amplification than Snapshot, at the cost of increased space +// amplification. While a Snapshot may increase write amplification across all +// flushes and compactions for the duration of its lifetime, an +// EventuallyFileOnlySnapshot only incurs that cost for flushes/compactions if +// memtables at the time of EFOS instantiation contained keys that the EFOS is +// interested in (i.e. its protectedRanges). In that case, the EFOS prevents +// elision of keys visible to it, similar to a Snapshot, until those memtables +// are flushed, and once that happens, the "EventuallyFileOnlySnapshot" +// transitions to a file-only snapshot state in which it pins zombies sstables +// like an open Iterator would, without pinning any memtables. Callers that can +// tolerate the increased space amplification of pinning zombie sstables until +// the snapshot is closed may prefer EventuallyFileOnlySnapshots for their +// reduced write amplification. Callers that desire the benefits of the file-only +// state that requires no pinning of memtables should call +// `WaitForFileOnlySnapshot()` (and possibly re-mint an EFOS if it returns +// ErrSnapshotExcised) before relying on the EFOS to keep producing iterators +// with zero write-amp and zero pinning of memtables in memory. +// +// EventuallyFileOnlySnapshots interact with the IngestAndExcise operation in +// subtle ways. No new iterators can be created once +// EventuallyFileOnlySnapshot.excised is set to true. +type EventuallyFileOnlySnapshot struct { + mu struct { + // NB: If both this mutex and db.mu are being grabbed, db.mu should be + // grabbed _before_ grabbing this one. + sync.Mutex + + // Either the snap field is set below, or the version is set at any given + // point of time. If a snapshot is referenced, this is not a file-only + // snapshot yet, and if a version is set (and ref'd) this is a file-only + // snapshot. + + // The wrapped regular snapshot, if not a file-only snapshot yet. + snap *Snapshot + // The wrapped version reference, if a file-only snapshot. + vers *version + } + + // Key ranges to watch for an excise on. + protectedRanges []KeyRange + // excised, if true, signals that the above ranges were excised during the + // lifetime of this snapshot. + excised atomic.Bool + + // The db the snapshot was created from. + db *DB + seqNum uint64 + + closed chan struct{} +} + +func (d *DB) makeEventuallyFileOnlySnapshot( + keyRanges []KeyRange, internalKeyRanges []internalKeyRange, +) *EventuallyFileOnlySnapshot { + isFileOnly := true + + d.mu.Lock() + defer d.mu.Unlock() + seqNum := d.mu.versions.visibleSeqNum.Load() + // Check if any of the keyRanges overlap with a memtable. + for i := range d.mu.mem.queue { + mem := d.mu.mem.queue[i] + if ingestMemtableOverlaps(d.cmp, mem, internalKeyRanges) { + isFileOnly = false + break + } + } + es := &EventuallyFileOnlySnapshot{ + db: d, + seqNum: seqNum, + protectedRanges: keyRanges, + closed: make(chan struct{}), + } + if isFileOnly { + es.mu.vers = d.mu.versions.currentVersion() + es.mu.vers.Ref() + } else { + s := &Snapshot{ + db: d, + seqNum: seqNum, + } + s.efos = es + es.mu.snap = s + d.mu.snapshots.pushBack(s) + } + return es +} + +// Transitions this EventuallyFileOnlySnapshot to a file-only snapshot. Requires +// earliestUnflushedSeqNum and vers to correspond to the same Version from the +// current or a past acquisition of db.mu. vers must have been Ref()'d before +// that mutex was released, if it was released. +// +// NB: The caller is expected to check for es.excised before making this +// call. +// +// d.mu must be held when calling this method. +func (es *EventuallyFileOnlySnapshot) transitionToFileOnlySnapshot(vers *version) error { + es.mu.Lock() + select { + case <-es.closed: + vers.UnrefLocked() + es.mu.Unlock() + return ErrClosed + default: + } + if es.mu.snap == nil { + es.mu.Unlock() + panic("pebble: tried to transition an eventually-file-only-snapshot twice") + } + // The caller has already called Ref() on vers. + es.mu.vers = vers + // NB: The callers should have already done a check of es.excised. + oldSnap := es.mu.snap + es.mu.snap = nil + es.mu.Unlock() + return oldSnap.closeLocked() +} + +// hasTransitioned returns true if this EFOS has transitioned to a file-only +// snapshot. +func (es *EventuallyFileOnlySnapshot) hasTransitioned() bool { + es.mu.Lock() + defer es.mu.Unlock() + return es.mu.vers != nil +} + +// waitForFlush waits for a flush on any memtables that need to be flushed +// before this EFOS can transition to a file-only snapshot. If this EFOS is +// waiting on a flush of the mutable memtable, it forces a rotation within +// `dur` duration. For immutable memtables, it schedules a flush and waits for +// it to finish. +func (es *EventuallyFileOnlySnapshot) waitForFlush(ctx context.Context, dur time.Duration) error { + es.db.mu.Lock() + defer es.db.mu.Unlock() + + earliestUnflushedSeqNum := es.db.getEarliestUnflushedSeqNumLocked() + for earliestUnflushedSeqNum < es.seqNum { + select { + case <-es.closed: + return ErrClosed + case <-ctx.Done(): + return ctx.Err() + default: + } + // Check if the current mutable memtable contains keys less than seqNum. + // If so, rotate it. + if es.db.mu.mem.mutable.logSeqNum < es.seqNum && dur.Nanoseconds() > 0 { + es.db.maybeScheduleDelayedFlush(es.db.mu.mem.mutable, dur) + } else { + // Find the last memtable that contains seqNums less than es.seqNum, + // and force a flush on it. + var mem *flushableEntry + for i := range es.db.mu.mem.queue { + if es.db.mu.mem.queue[i].logSeqNum < es.seqNum { + mem = es.db.mu.mem.queue[i] + } + } + mem.flushForced = true + es.db.maybeScheduleFlush() + } + es.db.mu.compact.cond.Wait() + + earliestUnflushedSeqNum = es.db.getEarliestUnflushedSeqNumLocked() + } + if es.excised.Load() { + return ErrSnapshotExcised + } + return nil +} + +// WaitForFileOnlySnapshot blocks the calling goroutine until this snapshot +// has been converted into a file-only snapshot (i.e. all memtables containing +// keys < seqNum are flushed). A duration can be passed in, and if nonzero, +// a delayed flush will be scheduled at that duration if necessary. +// +// Idempotent; can be called multiple times with no side effects. +func (es *EventuallyFileOnlySnapshot) WaitForFileOnlySnapshot( + ctx context.Context, dur time.Duration, +) error { + if es.hasTransitioned() { + return nil + } + + if err := es.waitForFlush(ctx, dur); err != nil { + return err + } + + if invariants.Enabled { + // Since we aren't returning an error, we _must_ have transitioned to a + // file-only snapshot by now. + if !es.hasTransitioned() { + panic("expected EFOS to have transitioned to file-only snapshot after flush") + } + } + return nil +} + +// Close closes the file-only snapshot and releases all referenced resources. +// Not idempotent. +func (es *EventuallyFileOnlySnapshot) Close() error { + close(es.closed) + es.db.mu.Lock() + defer es.db.mu.Unlock() + es.mu.Lock() + defer es.mu.Unlock() + + if es.mu.snap != nil { + if err := es.mu.snap.closeLocked(); err != nil { + return err + } + } + if es.mu.vers != nil { + es.mu.vers.UnrefLocked() + } + return nil +} + +// Get implements the Reader interface. +func (es *EventuallyFileOnlySnapshot) Get(key []byte) (value []byte, closer io.Closer, err error) { + // TODO(jackson): Use getInternal. + iter, err := es.NewIter(nil) + if err != nil { + return nil, nil, err + } + var valid bool + if es.db.opts.Comparer.Split != nil { + valid = iter.SeekPrefixGE(key) + } else { + valid = iter.SeekGE(key) + } + if !valid { + if err = firstError(iter.Error(), iter.Close()); err != nil { + return nil, nil, err + } + return nil, nil, ErrNotFound + } + if !es.db.equal(iter.Key(), key) { + return nil, nil, firstError(iter.Close(), ErrNotFound) + } + return iter.Value(), iter, nil +} + +// NewIter returns an iterator that is unpositioned (Iterator.Valid() will +// return false). The iterator can be positioned via a call to SeekGE, +// SeekLT, First or Last. +func (es *EventuallyFileOnlySnapshot) NewIter(o *IterOptions) (*Iterator, error) { + return es.NewIterWithContext(context.Background(), o) +} + +// NewIterWithContext is like NewIter, and additionally accepts a context for +// tracing. +func (es *EventuallyFileOnlySnapshot) NewIterWithContext( + ctx context.Context, o *IterOptions, +) (*Iterator, error) { + select { + case <-es.closed: + panic(ErrClosed) + default: + } + + es.mu.Lock() + defer es.mu.Unlock() + if es.mu.vers != nil { + sOpts := snapshotIterOpts{seqNum: es.seqNum, vers: es.mu.vers} + return es.db.newIter(ctx, nil /* batch */, sOpts, o), nil + } + + if es.excised.Load() { + return nil, ErrSnapshotExcised + } + sOpts := snapshotIterOpts{seqNum: es.seqNum} + iter := es.db.newIter(ctx, nil /* batch */, sOpts, o) + + // If excised is true, then keys relevant to the snapshot might not be + // present in the readState being used by the iterator. Error out. + if es.excised.Load() { + iter.Close() + return nil, ErrSnapshotExcised + } + return iter, nil +} + +// ScanInternal scans all internal keys within the specified bounds, truncating +// any rangedels and rangekeys to those bounds. For use when an external user +// needs to be aware of all internal keys that make up a key range. +// +// See comment on db.ScanInternal for the behaviour that can be expected of +// point keys deleted by range dels and keys masked by range keys. +func (es *EventuallyFileOnlySnapshot) ScanInternal( + ctx context.Context, + lower, upper []byte, + visitPointKey func(key *InternalKey, value LazyValue, iterInfo IteratorLevel) error, + visitRangeDel func(start, end []byte, seqNum uint64) error, + visitRangeKey func(start, end []byte, keys []rangekey.Key) error, + visitSharedFile func(sst *SharedSSTMeta) error, +) error { + if es.db == nil { + panic(ErrClosed) + } + if es.excised.Load() { + return ErrSnapshotExcised + } + var sOpts snapshotIterOpts + es.mu.Lock() + if es.mu.vers != nil { + sOpts = snapshotIterOpts{ + seqNum: es.seqNum, + vers: es.mu.vers, + } + } else { + sOpts = snapshotIterOpts{ + seqNum: es.seqNum, + } + } + es.mu.Unlock() + opts := &scanInternalOptions{ + IterOptions: IterOptions{ + KeyTypes: IterKeyTypePointsAndRanges, + LowerBound: lower, + UpperBound: upper, + }, + visitPointKey: visitPointKey, + visitRangeDel: visitRangeDel, + visitRangeKey: visitRangeKey, + visitSharedFile: visitSharedFile, + skipSharedLevels: visitSharedFile != nil, + } + iter := es.db.newInternalIter(sOpts, opts) + defer iter.close() + + // If excised is true, then keys relevant to the snapshot might not be + // present in the readState being used by the iterator. Error out. + if es.excised.Load() { + return ErrSnapshotExcised + } + + return scanInternalImpl(ctx, lower, upper, iter, opts) +} diff --git a/vendor/github.com/cockroachdb/pebble/sstable/block.go b/vendor/github.com/cockroachdb/pebble/sstable/block.go index 864bb68..c6345ea 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/block.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/block.go @@ -10,7 +10,6 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" - "github.com/cockroachdb/pebble/internal/cache" "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/manual" @@ -47,7 +46,10 @@ type blockWriter struct { // ensuring that any of the keys with the same prefix can be used to // assemble the full key when the prefix does change. restarts []uint32 - curKey []byte + // Do not read curKey directly from outside blockWriter since it can have + // the InternalKeyKindSSTableInternalObsoleteBit set. Use getCurKey() or + // getCurUserKey() instead. + curKey []byte // curValue excludes the optional prefix provided to // storeWithOptionalValuePrefix. curValue []byte @@ -79,6 +81,20 @@ const setHasSameKeyPrefixRestartMask uint32 = 1 << 31 const restartMaskLittleEndianHighByteWithoutSetHasSamePrefix byte = 0b0111_1111 const restartMaskLittleEndianHighByteOnlySetHasSamePrefix byte = 0b1000_0000 +func (w *blockWriter) getCurKey() InternalKey { + k := base.DecodeInternalKey(w.curKey) + k.Trailer = k.Trailer & trailerObsoleteMask + return k +} + +func (w *blockWriter) getCurUserKey() []byte { + n := len(w.curKey) - base.InternalTrailerLen + if n < 0 { + panic(errors.AssertionFailedf("corrupt key in blockWriter buffer")) + } + return w.curKey[:n:n] +} + // If !addValuePrefix, the valuePrefix is ignored. func (w *blockWriter) storeWithOptionalValuePrefix( keySize int, @@ -191,12 +207,21 @@ func (w *blockWriter) storeWithOptionalValuePrefix( func (w *blockWriter) add(key InternalKey, value []byte) { w.addWithOptionalValuePrefix( - key, value, len(key.UserKey), false, 0, false) + key, false, value, len(key.UserKey), false, 0, false) } // Callers that always set addValuePrefix to false should use add() instead. +// +// isObsolete indicates whether this key-value pair is obsolete in this +// sstable (only applicable when writing data blocks) -- see the comment in +// table.go and the longer one in format.go. addValuePrefix adds a 1 byte +// prefix to the value, specified in valuePrefix -- this is used for data +// blocks in TableFormatPebblev3 onwards for SETs (see the comment in +// format.go, with more details in value_block.go). setHasSameKeyPrefix is +// also used in TableFormatPebblev3 onwards for SETs. func (w *blockWriter) addWithOptionalValuePrefix( key InternalKey, + isObsolete bool, value []byte, maxSharedKeyLen int, addValuePrefix bool, @@ -210,6 +235,9 @@ func (w *blockWriter) addWithOptionalValuePrefix( w.curKey = make([]byte, 0, size*2) } w.curKey = w.curKey[:size] + if isObsolete { + key.Trailer = key.Trailer | trailerObsoleteBit + } key.Encode(w.curKey) w.storeWithOptionalValuePrefix( @@ -371,16 +399,16 @@ type blockIter struct { // For a block encoded with a restart interval of 1, cached and cachedBuf // will not be used as there are no prefix compressed entries between the // restart points. - cached []blockEntry - cachedBuf []byte - cacheHandle cache.Handle - // The first key in the block. This is used by the caller to set bounds + cached []blockEntry + cachedBuf []byte + handle bufferHandle // for block iteration for already loaded blocks. - firstKey InternalKey + firstUserKey []byte lazyValueHandling struct { vbr *valueBlockReader hasValuePrefix bool } + hideObsoletePoints bool } // blockIter implements the base.InternalIterator interface. @@ -388,14 +416,16 @@ var _ base.InternalIterator = (*blockIter)(nil) func newBlockIter(cmp Compare, block block) (*blockIter, error) { i := &blockIter{} - return i, i.init(cmp, block, 0) + return i, i.init(cmp, block, 0, false) } func (i *blockIter) String() string { return "block" } -func (i *blockIter) init(cmp Compare, block block, globalSeqNum uint64) error { +func (i *blockIter) init( + cmp Compare, block block, globalSeqNum uint64, hideObsoletePoints bool, +) error { numRestarts := int32(binary.LittleEndian.Uint32(block[len(block)-4:])) if numRestarts == 0 { return base.CorruptionErrorf("pebble/table: invalid table (block has no restart points)") @@ -408,6 +438,7 @@ func (i *blockIter) init(cmp Compare, block block, globalSeqNum uint64) error { i.data = block i.fullKey = i.fullKey[:0] i.val = nil + i.hideObsoletePoints = hideObsoletePoints i.clearCache() if i.restarts > 0 { if err := i.readFirstKey(); err != nil { @@ -415,15 +446,21 @@ func (i *blockIter) init(cmp Compare, block block, globalSeqNum uint64) error { } } else { // Block is empty. - i.firstKey = InternalKey{} + i.firstUserKey = nil } return nil } -func (i *blockIter) initHandle(cmp Compare, block cache.Handle, globalSeqNum uint64) error { - i.cacheHandle.Release() - i.cacheHandle = block - return i.init(cmp, block.Get(), globalSeqNum) +// NB: two cases of hideObsoletePoints: +// - Local sstable iteration: globalSeqNum will be set iff the sstable was +// ingested. +// - Foreign sstable iteration: globalSeqNum is always set. +func (i *blockIter) initHandle( + cmp Compare, block bufferHandle, globalSeqNum uint64, hideObsoletePoints bool, +) error { + i.handle.Release() + i.handle = block + return i.init(cmp, block.Get(), globalSeqNum, hideObsoletePoints) } func (i *blockIter) invalidate() { @@ -590,24 +627,27 @@ func (i *blockIter) readFirstKey() error { // Manually inlining base.DecodeInternalKey provides a 5-10% speedup on // BlockIter benchmarks. if n := len(firstKey) - 8; n >= 0 { - i.firstKey.Trailer = binary.LittleEndian.Uint64(firstKey[n:]) - i.firstKey.UserKey = firstKey[:n:n] - if i.globalSeqNum != 0 { - i.firstKey.SetSeqNum(i.globalSeqNum) - } + i.firstUserKey = firstKey[:n:n] } else { - i.firstKey.Trailer = uint64(InternalKeyKindInvalid) - i.firstKey.UserKey = nil + i.firstUserKey = nil return base.CorruptionErrorf("pebble/table: invalid firstKey in block") } return nil } -func (i *blockIter) decodeInternalKey(key []byte) { +// The sstable internal obsolete bit is set when writing a block and unset by +// blockIter, so no code outside block writing/reading code ever sees it. +const trailerObsoleteBit = uint64(base.InternalKeyKindSSTableInternalObsoleteBit) +const trailerObsoleteMask = (InternalKeySeqNumMax << 8) | uint64(base.InternalKeyKindSSTableInternalObsoleteMask) + +func (i *blockIter) decodeInternalKey(key []byte) (hiddenPoint bool) { // Manually inlining base.DecodeInternalKey provides a 5-10% speedup on // BlockIter benchmarks. if n := len(key) - 8; n >= 0 { - i.ikey.Trailer = binary.LittleEndian.Uint64(key[n:]) + trailer := binary.LittleEndian.Uint64(key[n:]) + hiddenPoint = i.hideObsoletePoints && + (trailer&trailerObsoleteBit != 0) + i.ikey.Trailer = trailer & trailerObsoleteMask i.ikey.UserKey = key[:n:n] if i.globalSeqNum != 0 { i.ikey.SetSeqNum(i.globalSeqNum) @@ -616,6 +656,7 @@ func (i *blockIter) decodeInternalKey(key []byte) { i.ikey.Trailer = uint64(InternalKeyKindInvalid) i.ikey.UserKey = nil } + return hiddenPoint } func (i *blockIter) clearCache() { @@ -640,6 +681,10 @@ func (i *blockIter) cacheEntry() { i.cachedBuf = append(i.cachedBuf, i.key...) } +func (i *blockIter) getFirstUserKey() []byte { + return i.firstUserKey +} + // SeekGE implements internalIterator.SeekGE, as documented in the pebble // package. func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, base.LazyValue) { @@ -647,7 +692,6 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba panic(errors.AssertionFailedf("invalidated blockIter used")) } - ikey := base.MakeSearchKey(key) i.clearCache() // Find the index of the smallest restart point whose key is > the key // sought; index will be numRestarts if there is no such restart point. @@ -702,23 +746,23 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba ptr = unsafe.Pointer(uintptr(ptr) + 5) } - // Manually inlining base.DecodeInternalKey provides a 5-10% speedup on - // BlockIter benchmarks. + // Manually inlining part of base.DecodeInternalKey provides a 5-10% + // speedup on BlockIter benchmarks. s := getBytes(ptr, int(v1)) - var k InternalKey + var k []byte if n := len(s) - 8; n >= 0 { - k.Trailer = binary.LittleEndian.Uint64(s[n:]) - k.UserKey = s[:n:n] - // NB: We can't have duplicate keys if the globalSeqNum != 0, so we - // leave the seqnum on this key as 0 as it won't affect our search - // since ikey has the maximum seqnum. - } else { - k.Trailer = uint64(InternalKeyKindInvalid) + k = s[:n:n] } + // Else k is invalid, and left as nil - if base.InternalCompare(i.cmp, ikey, k) >= 0 { + if i.cmp(key, k) > 0 { + // The search key is greater than the user key at this restart point. + // Search beyond this restart point, since we are trying to find the + // first restart point with a user key >= the search key. index = h + 1 // preserves f(i-1) == false } else { + // k >= search key, so prune everything after index (since index + // satisfies the property we are looking for). upper = h // preserves f(j) == true } } @@ -726,21 +770,25 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba // => answer is index. } - // Since keys are strictly increasing, if index > 0 then the restart point at - // index-1 will be the largest whose key is <= the key sought. If index == - // 0, then all keys in this block are larger than the key sought, and offset - // remains at zero. + // index is the first restart point with key >= search key. Define the keys + // between a restart point and the next restart point as belonging to that + // restart point. + // + // Since keys are strictly increasing, if index > 0 then the restart point + // at index-1 will be the first one that has some keys belonging to it that + // could be equal to the search key. If index == 0, then all keys in this + // block are larger than the key sought, and offset remains at zero. if index > 0 { i.offset = decodeRestart(i.data[i.restarts+4*(index-1):]) } i.readEntry() - i.decodeInternalKey(i.key) + hiddenPoint := i.decodeInternalKey(i.key) // Iterate from that restart point to somewhere >= the key sought. if !i.valid() { return nil, base.LazyValue{} } - if base.InternalCompare(i.cmp, i.ikey, ikey) >= 0 { + if !hiddenPoint && i.cmp(i.ikey.UserKey, key) >= 0 { // Initialize i.lazyValue if !i.lazyValueHandling.hasValuePrefix || base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet { @@ -753,7 +801,7 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba return &i.ikey, i.lazyValue } for i.Next(); i.valid(); i.Next() { - if base.InternalCompare(i.cmp, i.ikey, ikey) >= 0 { + if i.cmp(i.ikey.UserKey, key) >= 0 { // i.Next() has already initialized i.lazyValue. return &i.ikey, i.lazyValue } @@ -777,8 +825,6 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba panic(errors.AssertionFailedf("invalidated blockIter used")) } - ikey := base.MakeSearchKey(key) - i.clearCache() // Find the index of the smallest restart point whose key is >= the key // sought; index will be numRestarts if there is no such restart point. @@ -833,23 +879,23 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba ptr = unsafe.Pointer(uintptr(ptr) + 5) } - // Manually inlining base.DecodeInternalKey provides a 5-10% speedup on - // BlockIter benchmarks. + // Manually inlining part of base.DecodeInternalKey provides a 5-10% + // speedup on BlockIter benchmarks. s := getBytes(ptr, int(v1)) - var k InternalKey + var k []byte if n := len(s) - 8; n >= 0 { - k.Trailer = binary.LittleEndian.Uint64(s[n:]) - k.UserKey = s[:n:n] - // NB: We can't have duplicate keys if the globalSeqNum != 0, so we - // leave the seqnum on this key as 0 as it won't affect our search - // since ikey has the maximum seqnum. - } else { - k.Trailer = uint64(InternalKeyKindInvalid) + k = s[:n:n] } + // Else k is invalid, and left as nil - if base.InternalCompare(i.cmp, ikey, k) > 0 { + if i.cmp(key, k) > 0 { + // The search key is greater than the user key at this restart point. + // Search beyond this restart point, since we are trying to find the + // first restart point with a user key >= the search key. index = h + 1 // preserves f(i-1) == false } else { + // k >= search key, so prune everything after index (since index + // satisfies the property we are looking for). upper = h // preserves f(j) == true } } @@ -857,8 +903,15 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba // => answer is index. } - // Since keys are strictly increasing, if index > 0 then the restart point at - // index-1 will be the largest whose key is < the key sought. + // index is the first restart point with key >= search key. Define the keys + // between a restart point and the next restart point as belonging to that + // restart point. Note that index could be equal to i.numRestarts, i.e., we + // are past the last restart. + // + // Since keys are strictly increasing, if index > 0 then the restart point + // at index-1 will be the first one that has some keys belonging to it that + // are less than the search key. If index == 0, then all keys in this block + // are larger than the search key, so there is no match. targetOffset := i.restarts if index > 0 { i.offset = decodeRestart(i.data[i.restarts+4*(index-1):]) @@ -881,24 +934,41 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba for { i.offset = i.nextOffset i.readEntry() - i.decodeInternalKey(i.key) + // When hidden keys are common, there is additional optimization possible + // by not caching entries that are hidden (note that some calls to + // cacheEntry don't decode the internal key before caching, but checking + // whether a key is hidden does not require full decoding). However, we do + // need to use the blockEntry.offset in the cache for the first entry at + // the reset point to do the binary search when the cache is empty -- so + // we would need to cache that first entry (though not the key) even if + // was hidden. Our current assumption is that if there are large numbers + // of hidden keys we will be able to skip whole blocks (using block + // property filters) so we don't bother optimizing. + hiddenPoint := i.decodeInternalKey(i.key) - if i.cmp(i.ikey.UserKey, ikey.UserKey) >= 0 { + // NB: we don't use the hiddenPoint return value of decodeInternalKey + // since we want to stop as soon as we reach a key >= ikey.UserKey, so + // that we can reverse. + if i.cmp(i.ikey.UserKey, key) >= 0 { // The current key is greater than or equal to our search key. Back up to // the previous key which was less than our search key. Note that this for // loop will execute at least once with this if-block not being true, so // the key we are backing up to is the last one this loop cached. - i.Prev() - // i.Prev() has already initialized i.lazyValue. - return &i.ikey, i.lazyValue + return i.Prev() } if i.nextOffset >= targetOffset { - // We've reached the end of the current restart block. Return the current - // key. When the restart interval is 1, the first iteration of the for - // loop will bring us here. In that case ikey is backed by the block so - // we get the desired key stability guarantee for the lifetime of the - // blockIter. + // We've reached the end of the current restart block. Return the + // current key if not hidden, else call Prev(). + // + // When the restart interval is 1, the first iteration of the for loop + // will bring us here. In that case ikey is backed by the block so we + // get the desired key stability guarantee for the lifetime of the + // blockIter. That is, we never cache anything and therefore never + // return a key backed by cachedBuf. + if hiddenPoint { + return i.Prev() + } break } @@ -932,7 +1002,10 @@ func (i *blockIter) First() (*InternalKey, base.LazyValue) { } i.clearCache() i.readEntry() - i.decodeInternalKey(i.key) + hiddenPoint := i.decodeInternalKey(i.key) + if hiddenPoint { + return i.Next() + } if !i.lazyValueHandling.hasValuePrefix || base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet { i.lazyValue = base.MakeInPlaceValue(i.val) @@ -971,7 +1044,10 @@ func (i *blockIter) Last() (*InternalKey, base.LazyValue) { i.readEntry() } - i.decodeInternalKey(i.key) + hiddenPoint := i.decodeInternalKey(i.key) + if hiddenPoint { + return i.Prev() + } if !i.lazyValueHandling.hasValuePrefix || base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet { i.lazyValue = base.MakeInPlaceValue(i.val) @@ -1001,6 +1077,7 @@ func (i *blockIter) Next() (*InternalKey, base.LazyValue) { i.clearCache() } +start: i.offset = i.nextOffset if !i.valid() { return nil, base.LazyValue{} @@ -1008,11 +1085,17 @@ func (i *blockIter) Next() (*InternalKey, base.LazyValue) { i.readEntry() // Manually inlined version of i.decodeInternalKey(i.key). if n := len(i.key) - 8; n >= 0 { - i.ikey.Trailer = binary.LittleEndian.Uint64(i.key[n:]) + trailer := binary.LittleEndian.Uint64(i.key[n:]) + hiddenPoint := i.hideObsoletePoints && + (trailer&trailerObsoleteBit != 0) + i.ikey.Trailer = trailer & trailerObsoleteMask i.ikey.UserKey = i.key[:n:n] if i.globalSeqNum != 0 { i.ikey.SetSeqNum(i.globalSeqNum) } + if hiddenPoint { + goto start + } } else { i.ikey.Trailer = uint64(InternalKeyKindInvalid) i.ikey.UserKey = nil @@ -1028,8 +1111,8 @@ func (i *blockIter) Next() (*InternalKey, base.LazyValue) { return &i.ikey, i.lazyValue } -// nextPrefix is used for implementing NPrefix. -func (i *blockIter) nextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { +// NextPrefix implements (base.InternalIterator).NextPrefix. +func (i *blockIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { if i.lazyValueHandling.hasValuePrefix { return i.nextPrefixV3(succKey) } @@ -1154,6 +1237,7 @@ func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue) // The trailer is written in little endian, so the key kind is the first // byte in the trailer that is encoded in the slice [unshared-8:unshared]. keyKind := InternalKeyKind((*[manual.MaxArrayLen]byte)(ptr)[unshared-8]) + keyKind = keyKind & base.InternalKeyKindSSTableInternalObsoleteMask prefixChanged := false if keyKind == InternalKeyKindSet { if invariants.Enabled && value == 0 { @@ -1269,8 +1353,12 @@ func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue) i.key = i.fullKey } // Manually inlined version of i.decodeInternalKey(i.key). + hiddenPoint := false if n := len(i.key) - 8; n >= 0 { - i.ikey.Trailer = binary.LittleEndian.Uint64(i.key[n:]) + trailer := binary.LittleEndian.Uint64(i.key[n:]) + hiddenPoint = i.hideObsoletePoints && + (trailer&trailerObsoleteBit != 0) + i.ikey.Trailer = trailer & trailerObsoleteMask i.ikey.UserKey = i.key[:n:n] if i.globalSeqNum != 0 { i.ikey.SetSeqNum(i.globalSeqNum) @@ -1286,6 +1374,9 @@ func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue) } if prefixChanged || i.cmp(i.ikey.UserKey, succKey) >= 0 { // Prefix has changed. + if hiddenPoint { + return i.Next() + } if invariants.Enabled && !i.lazyValueHandling.hasValuePrefix { panic(errors.AssertionFailedf("nextPrefixV3 being run for non-v3 sstable")) } @@ -1307,23 +1398,11 @@ func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue) return i.SeekGE(succKey, base.SeekGEFlagsNone) } -// NextPrefix implements (base.InternalIterator).NextPrefix -func (i *blockIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { - const nextsBeforeSeek = 3 - k, v := i.Next() - for j := 1; k != nil && i.cmp(k.UserKey, succKey) < 0; j++ { - if j >= nextsBeforeSeek { - return i.SeekGE(succKey, base.SeekGEFlagsNone) - } - k, v = i.Next() - } - return k, v -} - // Prev implements internalIterator.Prev, as documented in the pebble // package. func (i *blockIter) Prev() (*InternalKey, base.LazyValue) { - if n := len(i.cached) - 1; n >= 0 { +start: + for n := len(i.cached) - 1; n >= 0; n-- { i.nextOffset = i.offset e := &i.cached[n] i.offset = e.offset @@ -1331,7 +1410,13 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) { // Manually inlined version of i.decodeInternalKey(i.key). i.key = i.cachedBuf[e.keyStart:e.keyEnd] if n := len(i.key) - 8; n >= 0 { - i.ikey.Trailer = binary.LittleEndian.Uint64(i.key[n:]) + trailer := binary.LittleEndian.Uint64(i.key[n:]) + hiddenPoint := i.hideObsoletePoints && + (trailer&trailerObsoleteBit != 0) + if hiddenPoint { + continue + } + i.ikey.Trailer = trailer & trailerObsoleteMask i.ikey.UserKey = i.key[:n:n] if i.globalSeqNum != 0 { i.ikey.SetSeqNum(i.globalSeqNum) @@ -1373,6 +1458,8 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) { // index ≤ h < upper offset := decodeRestart(i.data[i.restarts+4*h:]) if offset < targetOffset { + // Looking for the first restart that has offset >= targetOffset, so + // ignore h and earlier. index = h + 1 // preserves f(i-1) == false } else { upper = h // preserves f(j) == true @@ -1382,20 +1469,35 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) { // => answer is index. } + // index is first restart with offset >= targetOffset. Note that + // targetOffset may not be at a restart point since one can call Prev() + // after Next() (so the cache was not populated) and targetOffset refers to + // the current entry. index-1 must have an offset < targetOffset (it can't + // be equal to targetOffset since the binary search would have selected that + // as the index). i.offset = 0 if index > 0 { i.offset = decodeRestart(i.data[i.restarts+4*(index-1):]) } + // TODO(sumeer): why is the else case not an error given targetOffset is a + // valid offset. i.readEntry() + // We stop when i.nextOffset == targetOffset since the targetOffset is the + // entry we are stepping back from, and we don't need to cache the entry + // before it, since it is the candidate to return. for i.nextOffset < targetOffset { i.cacheEntry() i.offset = i.nextOffset i.readEntry() } - i.decodeInternalKey(i.key) + hiddenPoint := i.decodeInternalKey(i.key) + if hiddenPoint { + // Use the cache. + goto start + } if !i.lazyValueHandling.hasValuePrefix || base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet { i.lazyValue = base.MakeInPlaceValue(i.val) @@ -1425,8 +1527,8 @@ func (i *blockIter) Error() error { // Close implements internalIterator.Close, as documented in the pebble // package. func (i *blockIter) Close() error { - i.cacheHandle.Release() - i.cacheHandle = cache.Handle{} + i.handle.Release() + i.handle = bufferHandle{} i.val = nil i.lazyValue = base.LazyValue{} i.lazyValueHandling.vbr = nil @@ -1468,6 +1570,10 @@ type fragmentBlockIter struct { err error dir int8 closeHook func(i keyspan.FragmentIterator) error + + // elideSameSeqnum, if true, returns only the first-occurring (in forward + // order) Key for each sequence number. + elideSameSeqnum bool } func (i *fragmentBlockIter) resetForReuse() fragmentBlockIter { @@ -1496,6 +1602,24 @@ func (i *fragmentBlockIter) decodeSpanKeys(k *InternalKey, internalValue []byte) } } +func (i *fragmentBlockIter) elideKeysOfSameSeqNum() { + if invariants.Enabled { + if !i.elideSameSeqnum || len(i.span.Keys) == 0 { + panic("elideKeysOfSameSeqNum called when it should not be") + } + } + lastSeqNum := i.span.Keys[0].SeqNum() + k := 1 + for j := 1; j < len(i.span.Keys); j++ { + if lastSeqNum != i.span.Keys[j].SeqNum() { + lastSeqNum = i.span.Keys[j].SeqNum() + i.span.Keys[k] = i.span.Keys[j] + k++ + } + } + i.span.Keys = i.span.Keys[:k] +} + // gatherForward gathers internal keys with identical bounds. Keys defined over // spans of the keyspace are fragmented such that any overlapping key spans have // identical bounds. When these spans are persisted to a range deletion or range @@ -1545,6 +1669,9 @@ func (i *fragmentBlockIter) gatherForward(k *InternalKey, lazyValue base.LazyVal k, lazyValue = i.blockIter.Next() internalValue = lazyValue.InPlaceValue() } + if i.elideSameSeqnum && len(i.span.Keys) > 0 { + i.elideKeysOfSameSeqNum() + } // i.blockIter is positioned over the first internal key for the next span. return &i.span } @@ -1604,6 +1731,9 @@ func (i *fragmentBlockIter) gatherBackward(k *InternalKey, lazyValue base.LazyVa // Backwards iteration encounters internal keys in the wrong order. keyspan.SortKeysByTrailer(&i.span.Keys) + if i.elideSameSeqnum && len(i.span.Keys) > 0 { + i.elideKeysOfSameSeqNum() + } return &i.span } diff --git a/vendor/github.com/cockroachdb/pebble/sstable/block_property.go b/vendor/github.com/cockroachdb/pebble/sstable/block_property.go index 1e986fb..d85a2b2 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/block_property.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/block_property.go @@ -9,6 +9,7 @@ import ( "fmt" "math" "sync" + "unsafe" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/rangekey" @@ -117,7 +118,7 @@ type BlockPropertyCollector interface { } // SuffixReplaceableBlockCollector is an extension to the BlockPropertyCollector -// interface that allows a block property collector to indicate the it supports +// interface that allows a block property collector to indicate that it supports // being *updated* during suffix replacement, i.e. when an existing SST in which // all keys have the same key suffix is updated to have a new suffix. // @@ -189,13 +190,13 @@ type BoundLimitedBlockPropertyFilter interface { // indicates that the filter may be used to filter blocks that exclusively // contain keys ≥ `key`, so long as the blocks' keys also satisfy the upper // bound. - KeyIsWithinLowerBound(key *InternalKey) bool + KeyIsWithinLowerBound(key []byte) bool // KeyIsWithinUpperBound tests whether the provided internal key falls // within the current upper bound of the filter. A true return value // indicates that the filter may be used to filter blocks that exclusively // contain keys ≤ `key`, so long as the blocks' keys also satisfy the lower // bound. - KeyIsWithinUpperBound(key *InternalKey) bool + KeyIsWithinUpperBound(key []byte) bool } // BlockIntervalCollector is a helper implementation of BlockPropertyCollector @@ -643,15 +644,21 @@ func (f *BlockPropertiesFilterer) intersectsUserPropsAndFinishInit( // considered intersecting. continue } - byteProps := []byte(props) - if len(byteProps) < 1 { + if len(props) < 1 { return false, base.CorruptionErrorf( "block properties for %s is corrupted", f.filters[i].Name()) } - shortID := shortID(byteProps[0]) - intersects, err := f.filters[i].Intersects(byteProps[1:]) - if err != nil || !intersects { - return false, err + shortID := shortID(props[0]) + { + // Use an unsafe conversion to avoid allocating. Intersects() is not + // supposed to modify the given slice. + // Note that unsafe.StringData only works if the string is not empty + // (which we already checked). + byteProps := unsafe.Slice(unsafe.StringData(props), len(props)) + intersects, err := f.filters[i].Intersects(byteProps[1:]) + if err != nil || !intersects { + return false, err + } } // Intersects the sstable, so need to use this filter when // deciding whether to read blocks. @@ -684,12 +691,11 @@ func (f *BlockPropertiesFilterer) intersectsUserPropsAndFinishInit( // be unused within this file. return true, nil } - byteProps := []byte(props) - if len(byteProps) < 1 { + if len(props) < 1 { return false, base.CorruptionErrorf( "block properties for %s is corrupted", f.boundLimitedFilter.Name()) } - f.boundLimitedShortID = int(byteProps[0]) + f.boundLimitedShortID = int(props[0]) // We don't check for table-level intersection for the bound-limited filter. // The bound-limited filter is treated as vacuously intersecting. diff --git a/vendor/github.com/cockroachdb/pebble/sstable/buffer_pool.go b/vendor/github.com/cockroachdb/pebble/sstable/buffer_pool.go new file mode 100644 index 0000000..2e98d44 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/sstable/buffer_pool.go @@ -0,0 +1,148 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package sstable + +import ( + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/cache" +) + +// A bufferHandle is a handle to manually-managed memory. The handle may point +// to a block in the block cache (h.Get() != nil), or a buffer that exists +// outside the block cache allocated from a BufferPool (b.Valid()). +type bufferHandle struct { + h cache.Handle + b Buf +} + +// Get retrieves the underlying buffer referenced by the handle. +func (bh bufferHandle) Get() []byte { + if v := bh.h.Get(); v != nil { + return v + } else if bh.b.p != nil { + return bh.b.p.pool[bh.b.i].b + } + return nil +} + +// Release releases the buffer, either back to the block cache or BufferPool. +func (bh bufferHandle) Release() { + bh.h.Release() + bh.b.Release() +} + +// A BufferPool holds a pool of buffers for holding sstable blocks. An initial +// size of the pool is provided on Init, but a BufferPool will grow to meet the +// largest working set size. It'll never shrink. When a buffer is released, the +// BufferPool recycles the buffer for future allocations. +// +// A BufferPool should only be used for short-lived allocations with +// well-understood working set sizes to avoid excessive memory consumption. +// +// BufferPool is not thread-safe. +type BufferPool struct { + // pool contains all the buffers held by the pool, including buffers that + // are in-use. For every i < len(pool): pool[i].v is non-nil. + pool []allocedBuffer +} + +type allocedBuffer struct { + v *cache.Value + // b holds the current byte slice. It's backed by v, but may be a subslice + // of v's memory while the buffer is in-use [ len(b) ≤ len(v.Buf()) ]. + // + // If the buffer is not currently in-use, b is nil. When being recycled, the + // BufferPool.Alloc will reset b to be a subslice of v.Buf(). + b []byte +} + +// Init initializes the pool with an initial working set buffer size of +// `initialSize`. +func (p *BufferPool) Init(initialSize int) { + *p = BufferPool{ + pool: make([]allocedBuffer, 0, initialSize), + } +} + +// initPreallocated is like Init but for internal sstable package use in +// instances where a pre-allocated slice of []allocedBuffer already exists. It's +// used to avoid an extra allocation initializing BufferPool.pool. +func (p *BufferPool) initPreallocated(pool []allocedBuffer) { + *p = BufferPool{ + pool: pool[:0], + } +} + +// Release releases all buffers held by the pool and resets the pool to an +// uninitialized state. +func (p *BufferPool) Release() { + for i := range p.pool { + if p.pool[i].b != nil { + panic(errors.AssertionFailedf("Release called on a BufferPool with in-use buffers")) + } + cache.Free(p.pool[i].v) + } + *p = BufferPool{} +} + +// Alloc allocates a new buffer of size n. If the pool already holds a buffer at +// least as large as n, the pooled buffer is used instead. +// +// Alloc is O(MAX(N,M)) where N is the largest number of concurrently in-use +// buffers allocated and M is the initialSize passed to Init. +func (p *BufferPool) Alloc(n int) Buf { + unusableBufferIdx := -1 + for i := 0; i < len(p.pool); i++ { + if p.pool[i].b == nil { + if len(p.pool[i].v.Buf()) >= n { + p.pool[i].b = p.pool[i].v.Buf()[:n] + return Buf{p: p, i: i} + } + unusableBufferIdx = i + } + } + + // If we would need to grow the size of the pool to allocate another buffer, + // but there was a slot available occupied by a buffer that's just too + // small, replace the too-small buffer. + if len(p.pool) == cap(p.pool) && unusableBufferIdx >= 0 { + i := unusableBufferIdx + cache.Free(p.pool[i].v) + p.pool[i].v = cache.Alloc(n) + p.pool[i].b = p.pool[i].v.Buf() + return Buf{p: p, i: i} + } + + // Allocate a new buffer. + v := cache.Alloc(n) + p.pool = append(p.pool, allocedBuffer{v: v, b: v.Buf()[:n]}) + return Buf{p: p, i: len(p.pool) - 1} +} + +// A Buf holds a reference to a manually-managed, pooled byte buffer. +type Buf struct { + p *BufferPool + // i holds the index into p.pool where the buffer may be found. This scheme + // avoids needing to allocate the handle to the buffer on the heap at the + // cost of copying two words instead of one. + i int +} + +// Valid returns true if the buf holds a valid buffer. +func (b Buf) Valid() bool { + return b.p != nil +} + +// Release releases the buffer back to the pool. +func (b *Buf) Release() { + if b.p == nil { + return + } + // Clear the allocedBuffer's byte slice. This signals the allocated buffer + // is no longer in use and a future call to BufferPool.Alloc may reuse this + // buffer. + b.p.pool[b.i].b = nil + b.p = nil +} diff --git a/vendor/github.com/cockroachdb/pebble/sstable/compression.go b/vendor/github.com/cockroachdb/pebble/sstable/compression.go index 4dd5bea..0db70c8 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/compression.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/compression.go @@ -52,10 +52,10 @@ func decompressInto(blockType blockType, compressed []byte, buf []byte) ([]byte, return result, nil } -// decompressBlock decompresses an SST block, with space allocated from a cache. +// decompressBlock decompresses an SST block, with manually-allocated space. // NB: If decompressBlock returns (nil, nil), no decompression was necessary and // the caller may use `b` directly. -func decompressBlock(cache *cache.Cache, blockType blockType, b []byte) (*cache.Value, error) { +func decompressBlock(blockType blockType, b []byte) (*cache.Value, error) { if blockType == noCompressionBlockType { return nil, nil } diff --git a/vendor/github.com/cockroachdb/pebble/sstable/filter.go b/vendor/github.com/cockroachdb/pebble/sstable/filter.go index 2d73e12..7b2e1ab 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/filter.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/filter.go @@ -18,14 +18,32 @@ type FilterMetrics struct { Misses int64 } -var dummyFilterMetrics FilterMetrics +// FilterMetricsTracker is used to keep track of filter metrics. It contains the +// same metrics as FilterMetrics, but they can be updated atomically. An +// instance of FilterMetricsTracker can be passed to a Reader as a ReaderOption. +type FilterMetricsTracker struct { + // See FilterMetrics.Hits. + hits atomic.Int64 + // See FilterMetrics.Misses. + misses atomic.Int64 +} -func (m *FilterMetrics) readerApply(r *Reader) { +var _ ReaderOption = (*FilterMetricsTracker)(nil) + +func (m *FilterMetricsTracker) readerApply(r *Reader) { if r.tableFilter != nil { r.tableFilter.metrics = m } } +// Load returns the current values as FilterMetrics. +func (m *FilterMetricsTracker) Load() FilterMetrics { + return FilterMetrics{ + Hits: m.hits.Load(), + Misses: m.misses.Load(), + } +} + // BlockHandle is the file offset and length of a block. type BlockHandle struct { Offset, Length uint64 @@ -47,22 +65,24 @@ type filterWriter interface { type tableFilterReader struct { policy FilterPolicy - metrics *FilterMetrics + metrics *FilterMetricsTracker } func newTableFilterReader(policy FilterPolicy) *tableFilterReader { return &tableFilterReader{ policy: policy, - metrics: &dummyFilterMetrics, + metrics: nil, } } func (f *tableFilterReader) mayContain(data, key []byte) bool { mayContain := f.policy.MayContain(TableFilter, data, key) - if mayContain { - atomic.AddInt64(&f.metrics.Misses, 1) - } else { - atomic.AddInt64(&f.metrics.Hits, 1) + if f.metrics != nil { + if mayContain { + f.metrics.misses.Add(1) + } else { + f.metrics.hits.Add(1) + } } return mayContain } diff --git a/vendor/github.com/cockroachdb/pebble/sstable/format.go b/vendor/github.com/cockroachdb/pebble/sstable/format.go index a3c7300..82310a5 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/format.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/format.go @@ -23,15 +23,164 @@ const ( TableFormatRocksDBv2 TableFormatPebblev1 // Block properties. TableFormatPebblev2 // Range keys. - // TableFormatPebblev3 is not currently intended to subsume v2, as - // supporting value blocks adds a 1 byte prefix to each value. After - // thorough experimentation and some production experience, this may change. TableFormatPebblev3 // Value blocks. + TableFormatPebblev4 // DELSIZED tombstones. NumTableFormats - TableFormatMax = TableFormatPebblev3 + TableFormatMax = NumTableFormats - 1 ) +// TableFormatPebblev4, in addition to DELSIZED, introduces the use of +// InternalKeyKindSSTableInternalObsoleteBit. +// +// 1. Motivation +// +// We have various related problems caused by Pebble snapshots: +// +// - P1: RANGEDELs that delete points in the same sstable, but the points +// happen to not get deleted during compactions because of an open snapshot. +// This causes very expensive iteration, that has been observed in +// production deployments +// +// - P2: When iterating over a foreign sstable (in disaggregated storage), we +// need to do (a) point collapsing to expose at most one point per user key, +// (b) apply RANGEDELs in the sstable to hide deleted points in the same +// sstable. This per-sstable point collapsing iteration needs to be very +// efficient (ideally as efficient from a CPU perspective as iteration over +// regular sstables) since foreign sstables can be very long-lived -- one of +// the goals of disaggregated storage is to scale compute and disk bandwidth +// resources as a function of the hot (from a write perspective) data and +// not the whole data, so we don't want to have to rewrite foreign sstables +// solely to improve read performance. +// +// The ideal solution for P2 would allow user-facing reads to utilize the +// existing SST iterators (with slight modifications) and with no loss of +// efficiency. And for P1 and P2 we would like to skip whole blocks of +// overwritten/deleted points. Even when we can't skip whole blocks, avoiding +// key comparisons at iteration time to discover what points are deleted is +// very desirable, since keys can be long. +// +// We observe that: +// +// - Reads: +// - All user-facing reads in CockroachDB use iterators over the DB, hence +// have a higher read seqnum than all sstables (there are some rare cases +// that can violate this, but those are not important from a performance +// optimization perspective). +// +// - Certain internal-facing reads in CockroachDB use snapshots, but the +// snapshots are shortlived enough that most L5 and L6 sstables will have +// all seqnums lower than the snapshot seqnum. +// +// - Writes: +// - We already do key comparisons between points when writing the sstable +// to ensure that the sstable invariant (monotonically increasing internal +// keys) is not violated. So we know which points share the same userkey, +// and thereby which points are obsolete because there is a more recent +// point in the same sstable. +// +// - The compactionIter knows which point id deleted by a RANGEDEL even if +// the point does need to be written because of a snapshot. +// +// So this known information can be encoded in the sstable at write time and +// utilized for optimized reading. +// +// 2. Solution +// +// We primarily scope the solution to the following point kinds: SET, +// SETWITHDEL, DEL, DELSIZED, SINGLEDEL. These are the ones marked locally +// obsolete, i.e., obsolete within the sstable, and we can guarantee that at +// most one point will be exposed per user key. MERGE keys create more +// complexity: MERGE followed by MERGE causes multiple keys to not be +// obsolete. Same applies for MERGE followed by SET/SETWITHDEL/DEL*. Note +// that: +// +// - For regular sst iteration, the obsolete marking is a performance +// optimization, and multiple keys for the same userkey can be handled by +// higher layers in the iterator tree (specifically pebble.Iterator). +// +// - For foreign sst iteration, we disallow MERGEs to be written to such +// shared ssts (details below). +// +// The key kinds are marked with an obsolete bit +// (InternalKeyKindSSTableInternalObsoleteBit) when the key-value pair is +// obsolete. This marking is done within blockWriter, based on information +// passed to it by Writer. In turn, Writer uses a combination of key +// comparisons, and information provided by compactionIter to decide whether a +// key-value pair is obsolete. Additionally, a Pebble-internal +// BlockPropertyCollector (obsoleteKeyBlockPropertyCollector) is used to mark +// blocks where all key-value pairs are obsolete. Since the common case is +// non-obsolete blocks, this block property collector uses the empty byte +// slice to represent a non-obsolete block, which consumes no space in +// BlockHandleWithProperties.Props. +// +// At read time, the obsolete bit is only visible to the blockIter, which can +// be optionally configured to hide obsolete points. This hiding is only +// configured for data block iterators for sstables being read by user-facing +// iterators at a seqnum greater than the max seqnum in the sstable. +// Additionally, when this hiding is configured, a Pebble-internal block +// property filter (obsoleteKeyBlockPropertyFilter), is used to skip whole +// blocks that are obsolete. +// +// 2.1 Correctness +// +// Due to the level invariant, the sequence of seqnums for a user key in a +// sstable represents a contiguous subsequence of the seqnums for the userkey +// across the whole LSM, and is more recent than the seqnums in a sstable in a +// lower level. So exposing exactly one point from a sstable for a userkey +// will also mask the points for the userkey in lower levels. If we expose no +// point, because of RANGEDELs, that RANGEDEL will also mask the points in +// lower levels. +// +// Note that we do not need to do anything special at write time for +// SETWITHDEL and SINGLEDEL. This is because these key kinds are treated +// specially only by compactions, which do not hide obsolete points. For +// regular reads, SETWITHDEL behaves the same as SET and SINGLEDEL behaves the +// same as DEL. +// +// 2.2 Strictness and MERGE +// +// Setting the obsolete bit on point keys is advanced usage, so we support two +// modes, both of which must be truthful when setting the obsolete bit, but +// vary in when they don't set the obsolete bit. +// +// - Non-strict: In this mode, the bit does not need to be set for keys that +// are obsolete. Additionally, any sstable containing MERGE keys can only +// use this mode. An iterator over such an sstable, when configured to +// hideObsoletePoints, can expose multiple internal keys per user key, and +// can expose keys that are deleted by rangedels in the same sstable. This +// is the mode that non-advanced users should use. Pebble without +// disaggregated storage will also use this mode and will best-effort set +// the obsolete bit, to optimize iteration when snapshots have retained many +// obsolete keys. +// +// - Strict: In this mode, every obsolete key must have the obsolete bit set, +// and no MERGE keys are permitted. An iterator over such an sstable, when +// configured to hideObsoletePoints satisfies two properties: +// - S1: will expose at most one internal key per user key, which is the +// most recent one. +// - S2: will never expose keys that are deleted by rangedels in the same +// sstable. +// +// This is the mode for two use cases in disaggregated storage (which will +// exclude parts of the key space that has MERGEs), for levels that contain +// sstables that can become foreign sstables: +// - Pebble compaction output to these levels that can become foreign +// sstables. +// +// - CockroachDB ingest operations that can ingest into the levels that can +// become foreign sstables. Note, these are not sstables corresponding to +// copied data for CockroachDB range snapshots. This case occurs for +// operations like index backfills: these trivially satisfy the strictness +// criteria since they only write one key per userkey. +// +// TODO(sumeer): this latter case is not currently supported, since only +// Writer.AddWithForceObsolete calls are permitted for writing strict +// obsolete sstables. This is done to reduce the likelihood of bugs. One +// simple way to lift this limitation would be to disallow adding any +// RANGEDELs when a Pebble-external writer is trying to construct a strict +// obsolete sstable. + // ParseTableFormat parses the given magic bytes and version into its // corresponding internal TableFormat. func ParseTableFormat(magic []byte, version uint32) (TableFormat, error) { @@ -53,6 +202,8 @@ func ParseTableFormat(magic []byte, version uint32) (TableFormat, error) { return TableFormatPebblev2, nil case 3: return TableFormatPebblev3, nil + case 4: + return TableFormatPebblev4, nil default: return TableFormatUnspecified, base.CorruptionErrorf( "pebble/table: unsupported pebble format version %d", errors.Safe(version), @@ -78,6 +229,8 @@ func (f TableFormat) AsTuple() (string, uint32) { return pebbleDBMagic, 2 case TableFormatPebblev3: return pebbleDBMagic, 3 + case TableFormatPebblev4: + return pebbleDBMagic, 4 default: panic("sstable: unknown table format version tuple") } @@ -96,6 +249,8 @@ func (f TableFormat) String() string { return "(Pebble,v2)" case TableFormatPebblev3: return "(Pebble,v3)" + case TableFormatPebblev4: + return "(Pebble,v4)" default: panic("sstable: unknown table format version tuple") } diff --git a/vendor/github.com/cockroachdb/pebble/sstable/internal.go b/vendor/github.com/cockroachdb/pebble/sstable/internal.go index 9bab181..0fe7c99 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/internal.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/internal.go @@ -23,6 +23,8 @@ const ( InternalKeyKindLogData = base.InternalKeyKindLogData InternalKeyKindSingleDelete = base.InternalKeyKindSingleDelete InternalKeyKindRangeDelete = base.InternalKeyKindRangeDelete + InternalKeyKindSetWithDelete = base.InternalKeyKindSetWithDelete + InternalKeyKindDeleteSized = base.InternalKeyKindDeleteSized InternalKeyKindMax = base.InternalKeyKindMax InternalKeyKindInvalid = base.InternalKeyKindInvalid InternalKeySeqNumBatch = base.InternalKeySeqNumBatch diff --git a/vendor/github.com/cockroachdb/pebble/sstable/layout.go b/vendor/github.com/cockroachdb/pebble/sstable/layout.go new file mode 100644 index 0000000..bff1e30 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/sstable/layout.go @@ -0,0 +1,306 @@ +// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package sstable + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "io" + "sort" + "unsafe" + + "github.com/cockroachdb/pebble/internal/base" +) + +// Layout describes the block organization of an sstable. +type Layout struct { + // NOTE: changes to fields in this struct should also be reflected in + // ValidateBlockChecksums, which validates a static list of BlockHandles + // referenced in this struct. + + Data []BlockHandleWithProperties + Index []BlockHandle + TopIndex BlockHandle + Filter BlockHandle + RangeDel BlockHandle + RangeKey BlockHandle + ValueBlock []BlockHandle + ValueIndex BlockHandle + Properties BlockHandle + MetaIndex BlockHandle + Footer BlockHandle + Format TableFormat +} + +// Describe returns a description of the layout. If the verbose parameter is +// true, details of the structure of each block are returned as well. +func (l *Layout) Describe( + w io.Writer, verbose bool, r *Reader, fmtRecord func(key *base.InternalKey, value []byte), +) { + ctx := context.TODO() + type block struct { + BlockHandle + name string + } + var blocks []block + + for i := range l.Data { + blocks = append(blocks, block{l.Data[i].BlockHandle, "data"}) + } + for i := range l.Index { + blocks = append(blocks, block{l.Index[i], "index"}) + } + if l.TopIndex.Length != 0 { + blocks = append(blocks, block{l.TopIndex, "top-index"}) + } + if l.Filter.Length != 0 { + blocks = append(blocks, block{l.Filter, "filter"}) + } + if l.RangeDel.Length != 0 { + blocks = append(blocks, block{l.RangeDel, "range-del"}) + } + if l.RangeKey.Length != 0 { + blocks = append(blocks, block{l.RangeKey, "range-key"}) + } + for i := range l.ValueBlock { + blocks = append(blocks, block{l.ValueBlock[i], "value-block"}) + } + if l.ValueIndex.Length != 0 { + blocks = append(blocks, block{l.ValueIndex, "value-index"}) + } + if l.Properties.Length != 0 { + blocks = append(blocks, block{l.Properties, "properties"}) + } + if l.MetaIndex.Length != 0 { + blocks = append(blocks, block{l.MetaIndex, "meta-index"}) + } + if l.Footer.Length != 0 { + if l.Footer.Length == levelDBFooterLen { + blocks = append(blocks, block{l.Footer, "leveldb-footer"}) + } else { + blocks = append(blocks, block{l.Footer, "footer"}) + } + } + + sort.Slice(blocks, func(i, j int) bool { + return blocks[i].Offset < blocks[j].Offset + }) + + for i := range blocks { + b := &blocks[i] + fmt.Fprintf(w, "%10d %s (%d)\n", b.Offset, b.name, b.Length) + + if !verbose { + continue + } + if b.name == "filter" { + continue + } + + if b.name == "footer" || b.name == "leveldb-footer" { + trailer, offset := make([]byte, b.Length), b.Offset + _ = r.readable.ReadAt(ctx, trailer, int64(offset)) + + if b.name == "footer" { + checksumType := ChecksumType(trailer[0]) + fmt.Fprintf(w, "%10d checksum type: %s\n", offset, checksumType) + trailer, offset = trailer[1:], offset+1 + } + + metaHandle, n := binary.Uvarint(trailer) + metaLen, m := binary.Uvarint(trailer[n:]) + fmt.Fprintf(w, "%10d meta: offset=%d, length=%d\n", offset, metaHandle, metaLen) + trailer, offset = trailer[n+m:], offset+uint64(n+m) + + indexHandle, n := binary.Uvarint(trailer) + indexLen, m := binary.Uvarint(trailer[n:]) + fmt.Fprintf(w, "%10d index: offset=%d, length=%d\n", offset, indexHandle, indexLen) + trailer, offset = trailer[n+m:], offset+uint64(n+m) + + fmt.Fprintf(w, "%10d [padding]\n", offset) + + trailing := 12 + if b.name == "leveldb-footer" { + trailing = 8 + } + + offset += uint64(len(trailer) - trailing) + trailer = trailer[len(trailer)-trailing:] + + if b.name == "footer" { + version := trailer[:4] + fmt.Fprintf(w, "%10d version: %d\n", offset, binary.LittleEndian.Uint32(version)) + trailer, offset = trailer[4:], offset+4 + } + + magicNumber := trailer + fmt.Fprintf(w, "%10d magic number: 0x%x\n", offset, magicNumber) + + continue + } + + h, err := r.readBlock( + context.Background(), b.BlockHandle, nil /* transform */, nil /* readHandle */, nil /* stats */, nil /* buffer pool */) + if err != nil { + fmt.Fprintf(w, " [err: %s]\n", err) + continue + } + + getRestart := func(data []byte, restarts, i int32) int32 { + return decodeRestart(data[restarts+4*i:]) + } + + formatIsRestart := func(data []byte, restarts, numRestarts, offset int32) { + i := sort.Search(int(numRestarts), func(i int) bool { + return getRestart(data, restarts, int32(i)) >= offset + }) + if i < int(numRestarts) && getRestart(data, restarts, int32(i)) == offset { + fmt.Fprintf(w, " [restart]\n") + } else { + fmt.Fprintf(w, "\n") + } + } + + formatRestarts := func(data []byte, restarts, numRestarts int32) { + for i := int32(0); i < numRestarts; i++ { + offset := getRestart(data, restarts, i) + fmt.Fprintf(w, "%10d [restart %d]\n", + b.Offset+uint64(restarts+4*i), b.Offset+uint64(offset)) + } + } + + formatTrailer := func() { + trailer := make([]byte, blockTrailerLen) + offset := int64(b.Offset + b.Length) + _ = r.readable.ReadAt(ctx, trailer, offset) + bt := blockType(trailer[0]) + checksum := binary.LittleEndian.Uint32(trailer[1:]) + fmt.Fprintf(w, "%10d [trailer compression=%s checksum=0x%04x]\n", offset, bt, checksum) + } + + var lastKey InternalKey + switch b.name { + case "data", "range-del", "range-key": + iter, _ := newBlockIter(r.Compare, h.Get()) + for key, value := iter.First(); key != nil; key, value = iter.Next() { + ptr := unsafe.Pointer(uintptr(iter.ptr) + uintptr(iter.offset)) + shared, ptr := decodeVarint(ptr) + unshared, ptr := decodeVarint(ptr) + value2, _ := decodeVarint(ptr) + + total := iter.nextOffset - iter.offset + // The format of the numbers in the record line is: + // + // ( = [] + + ) + // + // is the total number of bytes for the record. + // is the size of the 3 varint encoded integers for , + // , and . + // is the number of key bytes shared with the previous key. + // is the number of unshared key bytes. + // is the number of value bytes. + fmt.Fprintf(w, "%10d record (%d = %d [%d] + %d + %d)", + b.Offset+uint64(iter.offset), total, + total-int32(unshared+value2), shared, unshared, value2) + formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset) + if fmtRecord != nil { + fmt.Fprintf(w, " ") + if l.Format < TableFormatPebblev3 { + fmtRecord(key, value.InPlaceValue()) + } else { + // InPlaceValue() will succeed even for data blocks where the + // actual value is in a different location, since this value was + // fetched from a blockIter which does not know about value + // blocks. + v := value.InPlaceValue() + if base.TrailerKind(key.Trailer) != InternalKeyKindSet { + fmtRecord(key, v) + } else if !isValueHandle(valuePrefix(v[0])) { + fmtRecord(key, v[1:]) + } else { + vh := decodeValueHandle(v[1:]) + fmtRecord(key, []byte(fmt.Sprintf("value handle %+v", vh))) + } + } + } + + if base.InternalCompare(r.Compare, lastKey, *key) >= 0 { + fmt.Fprintf(w, " WARNING: OUT OF ORDER KEYS!\n") + } + lastKey.Trailer = key.Trailer + lastKey.UserKey = append(lastKey.UserKey[:0], key.UserKey...) + } + formatRestarts(iter.data, iter.restarts, iter.numRestarts) + formatTrailer() + case "index", "top-index": + iter, _ := newBlockIter(r.Compare, h.Get()) + for key, value := iter.First(); key != nil; key, value = iter.Next() { + bh, err := decodeBlockHandleWithProperties(value.InPlaceValue()) + if err != nil { + fmt.Fprintf(w, "%10d [err: %s]\n", b.Offset+uint64(iter.offset), err) + continue + } + fmt.Fprintf(w, "%10d block:%d/%d", + b.Offset+uint64(iter.offset), bh.Offset, bh.Length) + formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset) + } + formatRestarts(iter.data, iter.restarts, iter.numRestarts) + formatTrailer() + case "properties": + iter, _ := newRawBlockIter(r.Compare, h.Get()) + for valid := iter.First(); valid; valid = iter.Next() { + fmt.Fprintf(w, "%10d %s (%d)", + b.Offset+uint64(iter.offset), iter.Key().UserKey, iter.nextOffset-iter.offset) + formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset) + } + formatRestarts(iter.data, iter.restarts, iter.numRestarts) + formatTrailer() + case "meta-index": + iter, _ := newRawBlockIter(r.Compare, h.Get()) + for valid := iter.First(); valid; valid = iter.Next() { + value := iter.Value() + var bh BlockHandle + var n int + var vbih valueBlocksIndexHandle + isValueBlocksIndexHandle := false + if bytes.Equal(iter.Key().UserKey, []byte(metaValueIndexName)) { + vbih, n, err = decodeValueBlocksIndexHandle(value) + bh = vbih.h + isValueBlocksIndexHandle = true + } else { + bh, n = decodeBlockHandle(value) + } + if n == 0 || n != len(value) { + fmt.Fprintf(w, "%10d [err: %s]\n", b.Offset+uint64(iter.offset), err) + continue + } + var vbihStr string + if isValueBlocksIndexHandle { + vbihStr = fmt.Sprintf(" value-blocks-index-lengths: %d(num), %d(offset), %d(length)", + vbih.blockNumByteLength, vbih.blockOffsetByteLength, vbih.blockLengthByteLength) + } + fmt.Fprintf(w, "%10d %s block:%d/%d%s", + b.Offset+uint64(iter.offset), iter.Key().UserKey, + bh.Offset, bh.Length, vbihStr) + formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset) + } + formatRestarts(iter.data, iter.restarts, iter.numRestarts) + formatTrailer() + case "value-block": + // We don't peer into the value-block since it can't be interpreted + // without the valueHandles. + case "value-index": + // We have already read the value-index to construct the list of + // value-blocks, so no need to do it again. + } + + h.Release() + } + + last := blocks[len(blocks)-1] + fmt.Fprintf(w, "%10d EOF\n", last.Offset+last.Length) +} diff --git a/vendor/github.com/cockroachdb/pebble/sstable/options.go b/vendor/github.com/cockroachdb/pebble/sstable/options.go index 66dbdd4..c5e1f79 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/options.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/options.go @@ -21,6 +21,17 @@ const ( NCompression ) +var ignoredInternalProperties = map[string]struct{}{ + "rocksdb.column.family.id": {}, + "rocksdb.fixed.key.length": {}, + "rocksdb.index.key.is.user.key": {}, + "rocksdb.index.value.is.delta.encoded": {}, + "rocksdb.oldest.key.time": {}, + "rocksdb.creation.time": {}, + "rocksdb.file.creation.time": {}, + "rocksdb.format.version": {}, +} + func (c Compression) String() string { switch c { case DefaultCompression: @@ -100,6 +111,9 @@ type ReaderOptions struct { // The default cache size is a zero-size cache. Cache *cache.Cache + // User properties specified in this map will not be added to sst.Properties.UserProperties. + DeniedUserProperties map[string]struct{} + // Comparer defines a total ordering over the space of []byte keys: a 'less // than' relationship. The same comparison algorithm must be used for reads // and writes over the lifetime of the DB. @@ -107,6 +121,9 @@ type ReaderOptions struct { // The default value uses the same ordering as bytes.Compare. Comparer *Comparer + // Merge defines the Merge function in use for this keyspace. + Merge base.Merge + // Filters is a map from filter policy name to filter policy. It is used for // debugging tools which may be used on multiple databases configured with // different filter policies. It is not necessary to populate this filters @@ -126,12 +143,18 @@ func (o ReaderOptions) ensureDefaults() ReaderOptions { if o.Comparer == nil { o.Comparer = base.DefaultComparer } + if o.Merge == nil { + o.Merge = base.DefaultMerger.Merge + } if o.MergerName == "" { o.MergerName = base.DefaultMerger.Name } if o.LoggerAndTracer == nil { o.LoggerAndTracer = base.NoopLoggerAndTracer{} } + if o.DeniedUserProperties == nil { + o.DeniedUserProperties = ignoredInternalProperties + } return o } @@ -210,6 +233,17 @@ type WriterOptions struct { // by a wider range of tools and libraries. TableFormat TableFormat + // IsStrictObsolete is only relevant for >= TableFormatPebblev4. See comment + // in format.go. Must be false if format < TableFormatPebblev4. + // + // TODO(bilal): set this when writing shared ssts. + IsStrictObsolete bool + + // WritingToLowestLevel is only relevant for >= TableFormatPebblev4. It is + // used to set the obsolete bit on DEL/DELSIZED/SINGLEDEL if they are the + // youngest for a userkey. + WritingToLowestLevel bool + // TablePropertyCollectors is a list of TablePropertyCollector creation // functions. A new TablePropertyCollector is created for each sstable built // and lives for the lifetime of the table. diff --git a/vendor/github.com/cockroachdb/pebble/sstable/properties.go b/vendor/github.com/cockroachdb/pebble/sstable/properties.go index 6be7f8d..3bbf34a 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/properties.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/properties.go @@ -23,20 +23,22 @@ var propTagMap = make(map[string]reflect.StructField) var propBoolTrue = []byte{'1'} var propBoolFalse = []byte{'0'} -var columnFamilyIDField = func() reflect.StructField { - f, ok := reflect.TypeOf(Properties{}).FieldByName("ColumnFamilyID") - if !ok { - panic("Properties.ColumnFamilyID field not found") - } - return f -}() - var propOffsetTagMap = make(map[uintptr]string) -func init() { - t := reflect.TypeOf(Properties{}) +func generateTagMaps(t reflect.Type, indexPrefix []int) { for i := 0; i < t.NumField(); i++ { f := t.Field(i) + if f.Type.Kind() == reflect.Struct { + if tag := f.Tag.Get("prop"); i == 0 && tag == "pebble.embbeded_common_properties" { + // CommonProperties struct embedded in Properties. Note that since + // CommonProperties is placed at the top of properties we can use + // the offsets of the fields within CommonProperties to determine + // the offsets of those fields within Properties. + generateTagMaps(f.Type, []int{i}) + continue + } + panic("pebble: unknown struct type in Properties") + } if tag := f.Tag.Get("prop"); tag != "" { switch f.Type.Kind() { case reflect.Bool: @@ -46,86 +48,125 @@ func init() { default: panic(fmt.Sprintf("unsupported property field type: %s %s", f.Name, f.Type)) } + if len(indexPrefix) > 0 { + // Prepend the index prefix so that we can use FieldByIndex on the top-level struct. + f.Index = append(indexPrefix[:len(indexPrefix):len(indexPrefix)], f.Index...) + } propTagMap[tag] = f propOffsetTagMap[f.Offset] = tag } } } -// Properties holds the sstable property values. The properties are -// automatically populated during sstable creation and load from the properties -// meta block when an sstable is opened. -type Properties struct { - // ID of column family for this SST file, corresponding to the CF identified - // by column_family_name. - ColumnFamilyID uint64 `prop:"rocksdb.column.family.id"` - // Name of the column family with which this SST file is associated. Empty if - // the column family is unknown. - ColumnFamilyName string `prop:"rocksdb.column.family.name"` - // The name of the comparer used in this table. - ComparerName string `prop:"rocksdb.comparator"` - // The compression algorithm used to compress blocks. - CompressionName string `prop:"rocksdb.compression"` - // The compression options used to compress blocks. - CompressionOptions string `prop:"rocksdb.compression_options"` - // The time when the SST file was created. Since SST files are immutable, - // this is equivalent to last modified time. - CreationTime uint64 `prop:"rocksdb.creation.time"` - // The total size of all data blocks. - DataSize uint64 `prop:"rocksdb.data.size"` - // The external sstable version format. Version 2 is the one RocksDB has been - // using since 5.13. RocksDB only uses the global sequence number for an - // sstable if this property has been set. - ExternalFormatVersion uint32 `prop:"rocksdb.external_sst_file.version"` - // Actual SST file creation time. 0 means unknown. - FileCreationTime uint64 `prop:"rocksdb.file.creation.time"` - // The name of the filter policy used in this table. Empty if no filter - // policy is used. - FilterPolicyName string `prop:"rocksdb.filter.policy"` - // The size of filter block. - FilterSize uint64 `prop:"rocksdb.filter.size"` - // If 0, key is variable length. Otherwise number of bytes for each key. - FixedKeyLen uint64 `prop:"rocksdb.fixed.key.length"` - // Format version, reserved for backward compatibility. - FormatVersion uint64 `prop:"rocksdb.format.version"` - // The global sequence number to use for all entries in the table. Present if - // the table was created externally and ingested whole. - GlobalSeqNum uint64 `prop:"rocksdb.external_sst_file.global_seqno"` - // Whether the index key is user key or an internal key. - IndexKeyIsUserKey uint64 `prop:"rocksdb.index.key.is.user.key"` - // Total number of index partitions if kTwoLevelIndexSearch is used. - IndexPartitions uint64 `prop:"rocksdb.index.partitions"` - // The size of index block. - IndexSize uint64 `prop:"rocksdb.index.size"` - // The index type. TODO(peter): add a more detailed description. - IndexType uint32 `prop:"rocksdb.block.based.table.index.type"` - // Whether delta encoding is used to encode the index values. - IndexValueIsDeltaEncoded uint64 `prop:"rocksdb.index.value.is.delta.encoded"` - // The name of the merger used in this table. Empty if no merger is used. - MergerName string `prop:"rocksdb.merge.operator"` - // The number of blocks in this table. - NumDataBlocks uint64 `prop:"rocksdb.num.data.blocks"` +func init() { + generateTagMaps(reflect.TypeOf(Properties{}), nil) +} + +// CommonProperties holds properties for either a virtual or a physical sstable. This +// can be used by code which doesn't care to make the distinction between physical +// and virtual sstables properties. +// +// For virtual sstables, fields are constructed through extrapolation upon virtual +// reader construction. See MakeVirtualReader for implementation details. +// +// NB: The values of these properties can affect correctness. For example, +// if NumRangeKeySets == 0, but the sstable actually contains range keys, then +// the iterators will behave incorrectly. +type CommonProperties struct { + // The number of entries in this table. + NumEntries uint64 `prop:"rocksdb.num.entries"` + // Total raw key size. + RawKeySize uint64 `prop:"rocksdb.raw.key.size"` + // Total raw value size. + RawValueSize uint64 `prop:"rocksdb.raw.value.size"` + // Total raw key size of point deletion tombstones. This value is comparable + // to RawKeySize. + RawPointTombstoneKeySize uint64 `prop:"pebble.raw.point-tombstone.key.size"` + // Sum of the raw value sizes carried by point deletion tombstones + // containing size estimates. See the DeleteSized key kind. This value is + // comparable to Raw{Key,Value}Size. + RawPointTombstoneValueSize uint64 `prop:"pebble.raw.point-tombstone.value.size"` + // The number of point deletion entries ("tombstones") in this table that + // carry a size hint indicating the size of the value the tombstone deletes. + NumSizedDeletions uint64 `prop:"pebble.num.deletions.sized"` // The number of deletion entries in this table, including both point and // range deletions. NumDeletions uint64 `prop:"rocksdb.deleted.keys"` - // The number of entries in this table. - NumEntries uint64 `prop:"rocksdb.num.entries"` - // The number of merge operands in the table. - NumMergeOperands uint64 `prop:"rocksdb.merge.operands"` // The number of range deletions in this table. NumRangeDeletions uint64 `prop:"rocksdb.num.range-deletions"` // The number of RANGEKEYDELs in this table. NumRangeKeyDels uint64 `prop:"pebble.num.range-key-dels"` // The number of RANGEKEYSETs in this table. NumRangeKeySets uint64 `prop:"pebble.num.range-key-sets"` + // Total size of value blocks and value index block. Only serialized if > 0. + ValueBlocksSize uint64 `prop:"pebble.value-blocks.size"` +} + +// String is only used for testing purposes. +func (c *CommonProperties) String() string { + var buf bytes.Buffer + v := reflect.ValueOf(*c) + loaded := make(map[uintptr]struct{}) + writeProperties(loaded, v, &buf) + return buf.String() +} + +// NumPointDeletions is the number of point deletions in the sstable. For virtual +// sstables, this is an estimate. +func (c *CommonProperties) NumPointDeletions() uint64 { + return c.NumDeletions - c.NumRangeDeletions +} + +// Properties holds the sstable property values. The properties are +// automatically populated during sstable creation and load from the properties +// meta block when an sstable is opened. +type Properties struct { + // CommonProperties needs to be at the top of the Properties struct so that the + // offsets of the fields in CommonProperties match the offsets of the embedded + // fields of CommonProperties in Properties. + CommonProperties `prop:"pebble.embbeded_common_properties"` + + // The name of the comparer used in this table. + ComparerName string `prop:"rocksdb.comparator"` + // The compression algorithm used to compress blocks. + CompressionName string `prop:"rocksdb.compression"` + // The compression options used to compress blocks. + CompressionOptions string `prop:"rocksdb.compression_options"` + // The total size of all data blocks. + DataSize uint64 `prop:"rocksdb.data.size"` + // The external sstable version format. Version 2 is the one RocksDB has been + // using since 5.13. RocksDB only uses the global sequence number for an + // sstable if this property has been set. + ExternalFormatVersion uint32 `prop:"rocksdb.external_sst_file.version"` + // The name of the filter policy used in this table. Empty if no filter + // policy is used. + FilterPolicyName string `prop:"rocksdb.filter.policy"` + // The size of filter block. + FilterSize uint64 `prop:"rocksdb.filter.size"` + // The global sequence number to use for all entries in the table. Present if + // the table was created externally and ingested whole. + GlobalSeqNum uint64 `prop:"rocksdb.external_sst_file.global_seqno"` + // Total number of index partitions if kTwoLevelIndexSearch is used. + IndexPartitions uint64 `prop:"rocksdb.index.partitions"` + // The size of index block. + IndexSize uint64 `prop:"rocksdb.index.size"` + // The index type. TODO(peter): add a more detailed description. + IndexType uint32 `prop:"rocksdb.block.based.table.index.type"` + // For formats >= TableFormatPebblev4, this is set to true if the obsolete + // bit is strict for all the point keys. + IsStrictObsolete bool `prop:"pebble.obsolete.is_strict"` + // The name of the merger used in this table. Empty if no merger is used. + MergerName string `prop:"rocksdb.merge.operator"` + // The number of blocks in this table. + NumDataBlocks uint64 `prop:"rocksdb.num.data.blocks"` + // The number of merge operands in the table. + NumMergeOperands uint64 `prop:"rocksdb.merge.operands"` // The number of RANGEKEYUNSETs in this table. NumRangeKeyUnsets uint64 `prop:"pebble.num.range-key-unsets"` // The number of value blocks in this table. Only serialized if > 0. NumValueBlocks uint64 `prop:"pebble.num.value-blocks"` // The number of values stored in value blocks. Only serialized if > 0. NumValuesInValueBlocks uint64 `prop:"pebble.num.values.in.value-blocks"` - // Timestamp of the earliest key. 0 if unknown. - OldestKeyTime uint64 `prop:"rocksdb.oldest.key.time"` // The name of the prefix extractor used in this table. Empty if no prefix // extractor is used. PrefixExtractorName string `prop:"rocksdb.prefix.extractor.name"` @@ -134,14 +175,10 @@ type Properties struct { // A comma separated list of names of the property collectors used in this // table. PropertyCollectorNames string `prop:"rocksdb.property.collectors"` - // Total raw key size. - RawKeySize uint64 `prop:"rocksdb.raw.key.size"` // Total raw rangekey key size. RawRangeKeyKeySize uint64 `prop:"pebble.raw.range-key.key.size"` // Total raw rangekey value size. RawRangeKeyValueSize uint64 `prop:"pebble.raw.range-key.value.size"` - // Total raw value size. - RawValueSize uint64 `prop:"rocksdb.raw.value.size"` // The total number of keys in this table that were pinned by open snapshots. SnapshotPinnedKeys uint64 `prop:"pebble.num.snapshot-pinned-keys"` // The cumulative bytes of keys in this table that were pinned by @@ -154,8 +191,6 @@ type Properties struct { TopLevelIndexSize uint64 `prop:"rocksdb.top-level.index.size"` // User collected properties. UserProperties map[string]string - // Total size of value blocks and value index block. Only serialized if > 0. - ValueBlocksSize uint64 `prop:"pebble.value-blocks.size"` // If filtering is enabled, was the filter created on the whole key. WholeKeyFiltering bool `prop:"rocksdb.block.based.table.whole.key.filtering"` @@ -176,12 +211,15 @@ func (p *Properties) NumRangeKeys() uint64 { return p.NumRangeKeyDels + p.NumRangeKeySets + p.NumRangeKeyUnsets } -func (p *Properties) String() string { - var buf bytes.Buffer - v := reflect.ValueOf(*p) +func writeProperties(loaded map[uintptr]struct{}, v reflect.Value, buf *bytes.Buffer) { vt := v.Type() for i := 0; i < v.NumField(); i++ { ft := vt.Field(i) + if ft.Type.Kind() == reflect.Struct { + // Embedded struct within the properties. + writeProperties(loaded, v.Field(i), buf) + continue + } tag := ft.Tag.Get("prop") if tag == "" { continue @@ -191,30 +229,33 @@ func (p *Properties) String() string { // TODO(peter): Use f.IsZero() when we can rely on go1.13. if zero := reflect.Zero(f.Type()); zero.Interface() == f.Interface() { // Skip printing of zero values which were not loaded from disk. - if _, ok := p.Loaded[ft.Offset]; !ok { + if _, ok := loaded[ft.Offset]; !ok { continue } } - fmt.Fprintf(&buf, "%s: ", tag) + fmt.Fprintf(buf, "%s: ", tag) switch ft.Type.Kind() { case reflect.Bool: - fmt.Fprintf(&buf, "%t\n", f.Bool()) + fmt.Fprintf(buf, "%t\n", f.Bool()) case reflect.Uint32: - fmt.Fprintf(&buf, "%d\n", f.Uint()) + fmt.Fprintf(buf, "%d\n", f.Uint()) case reflect.Uint64: - u := f.Uint() - if ft.Offset == columnFamilyIDField.Offset && u == math.MaxInt32 { - fmt.Fprintf(&buf, "-\n") - } else { - fmt.Fprintf(&buf, "%d\n", f.Uint()) - } + fmt.Fprintf(buf, "%d\n", f.Uint()) case reflect.String: - fmt.Fprintf(&buf, "%s\n", f.String()) + fmt.Fprintf(buf, "%s\n", f.String()) default: panic("not reached") } } +} + +func (p *Properties) String() string { + var buf bytes.Buffer + v := reflect.ValueOf(*p) + writeProperties(p.Loaded, v, &buf) + + // Write the UserProperties. keys := make([]string, 0, len(p.UserProperties)) for key := range p.UserProperties { keys = append(keys, key) @@ -226,16 +267,18 @@ func (p *Properties) String() string { return buf.String() } -func (p *Properties) load(b block, blockOffset uint64) error { +func (p *Properties) load( + b block, blockOffset uint64, deniedUserProperties map[string]struct{}, +) error { i, err := newRawBlockIter(bytes.Compare, b) if err != nil { return err } p.Loaded = make(map[uintptr]struct{}) v := reflect.ValueOf(p).Elem() + for valid := i.First(); valid; valid = i.Next() { - tag := intern.Bytes(i.Key().UserKey) - if f, ok := propTagMap[tag]; ok { + if f, ok := propTagMap[string(i.Key().UserKey)]; ok { p.Loaded[f.Offset] = struct{}{} field := v.FieldByIndex(f.Index) switch f.Type.Kind() { @@ -245,7 +288,7 @@ func (p *Properties) load(b block, blockOffset uint64) error { field.SetUint(uint64(binary.LittleEndian.Uint32(i.Value()))) case reflect.Uint64: var n uint64 - if tag == propGlobalSeqnumName { + if string(i.Key().UserKey) == propGlobalSeqnumName { n = binary.LittleEndian.Uint64(i.Value()) } else { n, _ = binary.Uvarint(i.Value()) @@ -261,7 +304,10 @@ func (p *Properties) load(b block, blockOffset uint64) error { if p.UserProperties == nil { p.UserProperties = make(map[string]string) } - p.UserProperties[tag] = string(i.Value()) + + if _, denied := deniedUserProperties[string(i.Key().UserKey)]; !denied { + p.UserProperties[intern.Bytes(i.Key().UserKey)] = string(i.Value()) + } } return nil } @@ -297,16 +343,12 @@ func (p *Properties) saveString(m map[string][]byte, offset uintptr, value strin m[propOffsetTagMap[offset]] = []byte(value) } -func (p *Properties) save(w *rawBlockWriter) { +func (p *Properties) save(tblFormat TableFormat, w *rawBlockWriter) { m := make(map[string][]byte) for k, v := range p.UserProperties { m[k] = []byte(v) } - p.saveUvarint(m, unsafe.Offsetof(p.ColumnFamilyID), p.ColumnFamilyID) - if p.ColumnFamilyName != "" { - p.saveString(m, unsafe.Offsetof(p.ColumnFamilyName), p.ColumnFamilyName) - } if p.ComparerName != "" { p.saveString(m, unsafe.Offsetof(p.ComparerName), p.ComparerName) } @@ -316,37 +358,45 @@ func (p *Properties) save(w *rawBlockWriter) { if p.CompressionOptions != "" { p.saveString(m, unsafe.Offsetof(p.CompressionOptions), p.CompressionOptions) } - p.saveUvarint(m, unsafe.Offsetof(p.CreationTime), p.CreationTime) p.saveUvarint(m, unsafe.Offsetof(p.DataSize), p.DataSize) if p.ExternalFormatVersion != 0 { p.saveUint32(m, unsafe.Offsetof(p.ExternalFormatVersion), p.ExternalFormatVersion) p.saveUint64(m, unsafe.Offsetof(p.GlobalSeqNum), p.GlobalSeqNum) } - if p.FileCreationTime > 0 { - p.saveUvarint(m, unsafe.Offsetof(p.FileCreationTime), p.FileCreationTime) - } if p.FilterPolicyName != "" { p.saveString(m, unsafe.Offsetof(p.FilterPolicyName), p.FilterPolicyName) } p.saveUvarint(m, unsafe.Offsetof(p.FilterSize), p.FilterSize) - p.saveUvarint(m, unsafe.Offsetof(p.FixedKeyLen), p.FixedKeyLen) - p.saveUvarint(m, unsafe.Offsetof(p.FormatVersion), p.FormatVersion) - p.saveUvarint(m, unsafe.Offsetof(p.IndexKeyIsUserKey), p.IndexKeyIsUserKey) if p.IndexPartitions != 0 { p.saveUvarint(m, unsafe.Offsetof(p.IndexPartitions), p.IndexPartitions) p.saveUvarint(m, unsafe.Offsetof(p.TopLevelIndexSize), p.TopLevelIndexSize) } p.saveUvarint(m, unsafe.Offsetof(p.IndexSize), p.IndexSize) p.saveUint32(m, unsafe.Offsetof(p.IndexType), p.IndexType) - p.saveUvarint(m, unsafe.Offsetof(p.IndexValueIsDeltaEncoded), p.IndexValueIsDeltaEncoded) + if p.IsStrictObsolete { + p.saveBool(m, unsafe.Offsetof(p.IsStrictObsolete), p.IsStrictObsolete) + } if p.MergerName != "" { p.saveString(m, unsafe.Offsetof(p.MergerName), p.MergerName) } p.saveUvarint(m, unsafe.Offsetof(p.NumDataBlocks), p.NumDataBlocks) p.saveUvarint(m, unsafe.Offsetof(p.NumEntries), p.NumEntries) p.saveUvarint(m, unsafe.Offsetof(p.NumDeletions), p.NumDeletions) + if p.NumSizedDeletions > 0 { + p.saveUvarint(m, unsafe.Offsetof(p.NumSizedDeletions), p.NumSizedDeletions) + } p.saveUvarint(m, unsafe.Offsetof(p.NumMergeOperands), p.NumMergeOperands) p.saveUvarint(m, unsafe.Offsetof(p.NumRangeDeletions), p.NumRangeDeletions) + // NB: We only write out some properties for Pebble formats. This isn't + // strictly necessary because unrecognized properties are interpreted as + // user-defined properties, however writing them prevents byte-for-byte + // equivalence with RocksDB files that some of our testing requires. + if p.RawPointTombstoneKeySize > 0 && tblFormat >= TableFormatPebblev1 { + p.saveUvarint(m, unsafe.Offsetof(p.RawPointTombstoneKeySize), p.RawPointTombstoneKeySize) + } + if p.RawPointTombstoneValueSize > 0 { + p.saveUvarint(m, unsafe.Offsetof(p.RawPointTombstoneValueSize), p.RawPointTombstoneValueSize) + } if p.NumRangeKeys() > 0 { p.saveUvarint(m, unsafe.Offsetof(p.NumRangeKeyDels), p.NumRangeKeyDels) p.saveUvarint(m, unsafe.Offsetof(p.NumRangeKeySets), p.NumRangeKeySets) @@ -360,7 +410,6 @@ func (p *Properties) save(w *rawBlockWriter) { if p.NumValuesInValueBlocks > 0 { p.saveUvarint(m, unsafe.Offsetof(p.NumValuesInValueBlocks), p.NumValuesInValueBlocks) } - p.saveUvarint(m, unsafe.Offsetof(p.OldestKeyTime), p.OldestKeyTime) if p.PrefixExtractorName != "" { p.saveString(m, unsafe.Offsetof(p.PrefixExtractorName), p.PrefixExtractorName) } @@ -380,6 +429,16 @@ func (p *Properties) save(w *rawBlockWriter) { } p.saveBool(m, unsafe.Offsetof(p.WholeKeyFiltering), p.WholeKeyFiltering) + if tblFormat < TableFormatPebblev1 { + m["rocksdb.column.family.id"] = binary.AppendUvarint([]byte(nil), math.MaxInt32) + m["rocksdb.fixed.key.length"] = []byte{0x00} + m["rocksdb.index.key.is.user.key"] = []byte{0x00} + m["rocksdb.index.value.is.delta.encoded"] = []byte{0x00} + m["rocksdb.oldest.key.time"] = []byte{0x00} + m["rocksdb.creation.time"] = []byte{0x00} + m["rocksdb.format.version"] = []byte{0x00} + } + keys := make([]string, 0, len(m)) for key := range m { keys = append(keys, key) diff --git a/vendor/github.com/cockroachdb/pebble/sstable/reader.go b/vendor/github.com/cockroachdb/pebble/sstable/reader.go index 3c9aa38..d86a424 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/reader.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/reader.go @@ -8,13 +8,10 @@ import ( "bytes" "context" "encoding/binary" - "fmt" "io" "os" "sort" - "sync" "time" - "unsafe" "github.com/cespare/xxhash/v2" "github.com/cockroachdb/errors" @@ -26,6 +23,7 @@ import ( "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/private" "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing" ) var errCorruptIndexEntry = base.CorruptionErrorf("pebble/table: corrupt index entry") @@ -78,408 +76,6 @@ func encodeBlockHandleWithProperties(dst []byte, b BlockHandleWithProperties) [] // over those pairs. type block []byte -// Iterator iterates over an entire table of data. -type Iterator interface { - base.InternalIterator - - // NextPrefix implements (base.InternalIterator).NextPrefix. - NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) - - // MaybeFilteredKeys may be called when an iterator is exhausted to indicate - // whether or not the last positioning method may have skipped any keys due - // to block-property filters. This is used by the Pebble levelIter to - // control when an iterator steps to the next sstable. - // - // MaybeFilteredKeys may always return false positives, that is it may - // return true when no keys were filtered. It should only be called when the - // iterator is exhausted. It must never return false negatives when the - // iterator is exhausted. - MaybeFilteredKeys() bool - - SetCloseHook(fn func(i Iterator) error) -} - -// Iterator positioning optimizations and singleLevelIterator and -// twoLevelIterator: -// -// An iterator is absolute positioned using one of the Seek or First or Last -// calls. After absolute positioning, there can be relative positioning done -// by stepping using Prev or Next. -// -// We implement optimizations below where an absolute positioning call can in -// some cases use the current position to do less work. To understand these, -// we first define some terms. An iterator is bounds-exhausted if the bounds -// (upper of lower) have been reached. An iterator is data-exhausted if it has -// the reached the end of the data (forward or reverse) in the sstable. A -// singleLevelIterator only knows a local-data-exhausted property since when -// it is used as part of a twoLevelIterator, the twoLevelIterator can step to -// the next lower-level index block. -// -// The bounds-exhausted property is tracked by -// singleLevelIterator.exhaustedBounds being +1 (upper bound reached) or -1 -// (lower bound reached). The same field is reused by twoLevelIterator. Either -// may notice the exhaustion of the bound and set it. Note that if -// singleLevelIterator sets this property, it is not a local property (since -// the bound has been reached regardless of whether this is in the context of -// the twoLevelIterator or not). -// -// The data-exhausted property is tracked in a more subtle manner. We define -// two predicates: -// - partial-local-data-exhausted (PLDE): -// i.data.isDataInvalidated() || !i.data.valid() -// - partial-global-data-exhausted (PGDE): -// i.index.isDataInvalidated() || !i.index.valid() || i.data.isDataInvalidated() || -// !i.data.valid() -// -// PLDE is defined for a singleLevelIterator. PGDE is defined for a -// twoLevelIterator. Oddly, in our code below the singleLevelIterator does not -// know when it is part of a twoLevelIterator so it does not know when its -// property is local or global. -// -// Now to define data-exhausted: -// - Prerequisite: we must know that the iterator has been positioned and -// i.err is nil. -// - bounds-exhausted must not be true: -// If bounds-exhausted is true, we have incomplete knowledge of -// data-exhausted since PLDE or PGDE could be true because we could have -// chosen not to load index block or data block and figured out that the -// bound is exhausted (due to block property filters filtering out index and -// data blocks and going past the bound on the top level index block). Note -// that if we tried to separate out the BPF case from others we could -// develop more knowledge here. -// - PGDE is true for twoLevelIterator. PLDE is true if it is a standalone -// singleLevelIterator. !PLDE or !PGDE of course imply that data-exhausted -// is not true. -// -// An implication of the above is that if we are going to somehow utilize -// knowledge of data-exhausted in an optimization, we must not forget the -// existing value of bounds-exhausted since by forgetting the latter we can -// erroneously think that data-exhausted is true. Bug #2036 was due to this -// forgetting. -// -// Now to the two categories of optimizations we currently have: -// - Monotonic bounds optimization that reuse prior iterator position when -// doing seek: These only work with !data-exhausted. We could choose to make -// these work with data-exhausted but have not bothered because in the -// context of a DB if data-exhausted were true, the DB would move to the -// next file in the level. Note that this behavior of moving to the next -// file is not necessarily true for L0 files, so there could be some benefit -// in the future in this optimization. See the WARNING-data-exhausted -// comments if trying to optimize this in the future. -// - TrySeekUsingNext optimizations: these work regardless of exhaustion -// state. -// -// Implementation detail: In the code PLDE only checks that -// i.data.isDataInvalidated(). This narrower check is safe, since this is a -// subset of the set expressed by the OR expression. Also, it is not a -// de-optimization since whenever we exhaust the iterator we explicitly call -// i.data.invalidate(). PGDE checks i.index.isDataInvalidated() && -// i.data.isDataInvalidated(). Again, this narrower check is safe, and not a -// de-optimization since whenever we exhaust the iterator we explicitly call -// i.index.invalidate() and i.data.invalidate(). The && is questionable -- for -// now this is a bit of defensive code. We should seriously consider removing -// it, since defensive code suggests we are not confident about our invariants -// (and if we are not confident, we need more invariant assertions, not -// defensive code). -// -// TODO(sumeer): remove the aforementioned defensive code. - -// singleLevelIterator iterates over an entire table of data. To seek for a given -// key, it first looks in the index for the block that contains that key, and then -// looks inside that block. -type singleLevelIterator struct { - ctx context.Context - cmp Compare - // Global lower/upper bound for the iterator. - lower []byte - upper []byte - bpfs *BlockPropertiesFilterer - // Per-block lower/upper bound. Nil if the bound does not apply to the block - // because we determined the block lies completely within the bound. - blockLower []byte - blockUpper []byte - reader *Reader - index blockIter - data blockIter - dataRH objstorage.ReadHandle - dataRHPrealloc objstorage.PreallocatedReadHandle - // dataBH refers to the last data block that the iterator considered - // loading. It may not actually have loaded the block, due to an error or - // because it was considered irrelevant. - dataBH BlockHandle - vbReader *valueBlockReader - // vbRH is the read handle for value blocks, which are in a different - // part of the sstable than data blocks. - vbRH objstorage.ReadHandle - vbRHPrealloc objstorage.PreallocatedReadHandle - err error - closeHook func(i Iterator) error - stats *base.InternalIteratorStats - - // boundsCmp and positionedUsingLatestBounds are for optimizing iteration - // that uses multiple adjacent bounds. The seek after setting a new bound - // can use the fact that the iterator is either within the previous bounds - // or exactly one key before or after the bounds. If the new bounds is - // after/before the previous bounds, and we are already positioned at a - // block that is relevant for the new bounds, we can try to first position - // using Next/Prev (repeatedly) instead of doing a more expensive seek. - // - // When there are wide files at higher levels that match the bounds - // but don't have any data for the bound, we will already be - // positioned at the key beyond the bounds and won't need to do much - // work -- given that most data is in L6, such files are likely to - // dominate the performance of the mergingIter, and may be the main - // benefit of this performance optimization (of course it also helps - // when the file that has the data has successive seeks that stay in - // the same block). - // - // Specifically, boundsCmp captures the relationship between the previous - // and current bounds, if the iterator had been positioned after setting - // the previous bounds. If it was not positioned, i.e., Seek/First/Last - // were not called, we don't know where it is positioned and cannot - // optimize. - // - // Example: Bounds moving forward, and iterator exhausted in forward direction. - // bounds = [f, h), ^ shows block iterator position - // file contents [ a b c d e f g h i j k ] - // ^ - // new bounds = [j, k). Since positionedUsingLatestBounds=true, boundsCmp is - // set to +1. SeekGE(j) can use next (the optimization also requires that j - // is within the block, but that is not for correctness, but to limit the - // optimization to when it will actually be an optimization). - // - // Example: Bounds moving forward. - // bounds = [f, h), ^ shows block iterator position - // file contents [ a b c d e f g h i j k ] - // ^ - // new bounds = [j, k). Since positionedUsingLatestBounds=true, boundsCmp is - // set to +1. SeekGE(j) can use next. - // - // Example: Bounds moving forward, but iterator not positioned using previous - // bounds. - // bounds = [f, h), ^ shows block iterator position - // file contents [ a b c d e f g h i j k ] - // ^ - // new bounds = [i, j). Iterator is at j since it was never positioned using - // [f, h). So positionedUsingLatestBounds=false, and boundsCmp is set to 0. - // SeekGE(i) will not use next. - // - // Example: Bounds moving forward and sparse file - // bounds = [f, h), ^ shows block iterator position - // file contents [ a z ] - // ^ - // new bounds = [j, k). Since positionedUsingLatestBounds=true, boundsCmp is - // set to +1. SeekGE(j) notices that the iterator is already past j and does - // not need to do anything. - // - // Similar examples can be constructed for backward iteration. - // - // This notion of exactly one key before or after the bounds is not quite - // true when block properties are used to ignore blocks. In that case we - // can't stop precisely at the first block that is past the bounds since - // we are using the index entries to enforce the bounds. - // - // e.g. 3 blocks with keys [b, c] [f, g], [i, j, k] with index entries d, - // h, l. And let the lower bound be k, and we are reverse iterating. If - // the block [i, j, k] is ignored due to the block interval annotations we - // do need to move the index to block [f, g] since the index entry for the - // [i, j, k] block is l which is not less than the lower bound of k. So we - // have passed the entries i, j. - // - // This behavior is harmless since the block property filters are fixed - // for the lifetime of the iterator so i, j are irrelevant. In addition, - // the current code will not load the [f, g] block, so the seek - // optimization that attempts to use Next/Prev do not apply anyway. - boundsCmp int - positionedUsingLatestBounds bool - - // exhaustedBounds represents whether the iterator is exhausted for - // iteration by reaching the upper or lower bound. +1 when exhausted - // the upper bound, -1 when exhausted the lower bound, and 0 when - // neither. exhaustedBounds is also used for the TrySeekUsingNext - // optimization in twoLevelIterator and singleLevelIterator. Care should be - // taken in setting this in twoLevelIterator before calling into - // singleLevelIterator, given that these two iterators share this field. - exhaustedBounds int8 - - // maybeFilteredKeysSingleLevel indicates whether the last iterator - // positioning operation may have skipped any data blocks due to - // block-property filters when positioning the index. - maybeFilteredKeysSingleLevel bool - - // useFilter specifies whether the filter block in this sstable, if present, - // should be used for prefix seeks or not. In some cases it is beneficial - // to skip a filter block even if it exists (eg. if probability of a match - // is high). - useFilter bool - lastBloomFilterMatched bool -} - -// singleLevelIterator implements the base.InternalIterator interface. -var _ base.InternalIterator = (*singleLevelIterator)(nil) - -var singleLevelIterPool = sync.Pool{ - New: func() interface{} { - i := &singleLevelIterator{} - // Note: this is a no-op if invariants are disabled or race is enabled. - invariants.SetFinalizer(i, checkSingleLevelIterator) - return i - }, -} - -var twoLevelIterPool = sync.Pool{ - New: func() interface{} { - i := &twoLevelIterator{} - // Note: this is a no-op if invariants are disabled or race is enabled. - invariants.SetFinalizer(i, checkTwoLevelIterator) - return i - }, -} - -// TODO(jackson): rangedel fragmentBlockIters can't be pooled because of some -// code paths that double Close the iters. Fix the double close and pool the -// *fragmentBlockIter type directly. - -var rangeKeyFragmentBlockIterPool = sync.Pool{ - New: func() interface{} { - i := &rangeKeyFragmentBlockIter{} - // Note: this is a no-op if invariants are disabled or race is enabled. - invariants.SetFinalizer(i, checkRangeKeyFragmentBlockIterator) - return i - }, -} - -func checkSingleLevelIterator(obj interface{}) { - i := obj.(*singleLevelIterator) - if p := i.data.cacheHandle.Get(); p != nil { - fmt.Fprintf(os.Stderr, "singleLevelIterator.data.cacheHandle is not nil: %p\n", p) - os.Exit(1) - } - if p := i.index.cacheHandle.Get(); p != nil { - fmt.Fprintf(os.Stderr, "singleLevelIterator.index.cacheHandle is not nil: %p\n", p) - os.Exit(1) - } -} - -func checkTwoLevelIterator(obj interface{}) { - i := obj.(*twoLevelIterator) - if p := i.data.cacheHandle.Get(); p != nil { - fmt.Fprintf(os.Stderr, "singleLevelIterator.data.cacheHandle is not nil: %p\n", p) - os.Exit(1) - } - if p := i.index.cacheHandle.Get(); p != nil { - fmt.Fprintf(os.Stderr, "singleLevelIterator.index.cacheHandle is not nil: %p\n", p) - os.Exit(1) - } -} - -func checkRangeKeyFragmentBlockIterator(obj interface{}) { - i := obj.(*rangeKeyFragmentBlockIter) - if p := i.blockIter.cacheHandle.Get(); p != nil { - fmt.Fprintf(os.Stderr, "fragmentBlockIter.blockIter.cacheHandle is not nil: %p\n", p) - os.Exit(1) - } -} - -// init initializes a singleLevelIterator for reading from the table. It is -// synonmous with Reader.NewIter, but allows for reusing of the iterator -// between different Readers. -func (i *singleLevelIterator) init( - ctx context.Context, - r *Reader, - lower, upper []byte, - filterer *BlockPropertiesFilterer, - useFilter bool, - stats *base.InternalIteratorStats, - rp ReaderProvider, -) error { - if r.err != nil { - return r.err - } - indexH, err := r.readIndex(ctx, stats) - if err != nil { - return err - } - - i.ctx = ctx - i.lower = lower - i.upper = upper - i.bpfs = filterer - i.useFilter = useFilter - i.reader = r - i.cmp = r.Compare - i.stats = stats - err = i.index.initHandle(i.cmp, indexH, r.Properties.GlobalSeqNum) - if err != nil { - // blockIter.Close releases indexH and always returns a nil error - _ = i.index.Close() - return err - } - i.dataRH = objstorage.UsePreallocatedReadHandle(r.readable, &i.dataRHPrealloc) - if r.tableFormat == TableFormatPebblev3 { - if r.Properties.NumValueBlocks > 0 { - // NB: we cannot avoid this ~248 byte allocation, since valueBlockReader - // can outlive the singleLevelIterator due to be being embedded in a - // LazyValue. This consumes ~2% in microbenchmark CPU profiles, but we - // should only optimize this if it shows up as significant in end-to-end - // CockroachDB benchmarks, since it is tricky to do so. One possibility - // is that if many sstable iterators only get positioned at latest - // versions of keys, and therefore never expose a LazyValue that is - // separated to their callers, they can put this valueBlockReader into a - // sync.Pool. - i.vbReader = &valueBlockReader{ - ctx: ctx, - bpOpen: i, - rp: rp, - vbih: r.valueBIH, - stats: stats, - } - i.data.lazyValueHandling.vbr = i.vbReader - i.vbRH = objstorage.UsePreallocatedReadHandle(r.readable, &i.vbRHPrealloc) - } - i.data.lazyValueHandling.hasValuePrefix = true - } - return nil -} - -// setupForCompaction sets up the singleLevelIterator for use with compactionIter. -// Currently, it skips readahead ramp-up. It should be called after init is called. -func (i *singleLevelIterator) setupForCompaction() { - i.dataRH.MaxReadahead() - if i.vbRH != nil { - i.vbRH.MaxReadahead() - } -} - -func (i *singleLevelIterator) resetForReuse() singleLevelIterator { - return singleLevelIterator{ - index: i.index.resetForReuse(), - data: i.data.resetForReuse(), - } -} - -func (i *singleLevelIterator) initBounds() { - // Trim the iteration bounds for the current block. We don't have to check - // the bounds on each iteration if the block is entirely contained within the - // iteration bounds. - i.blockLower = i.lower - if i.blockLower != nil { - key, _ := i.data.First() - if key != nil && i.cmp(i.blockLower, key.UserKey) < 0 { - // The lower-bound is less than the first key in the block. No need - // to check the lower-bound again for this block. - i.blockLower = nil - } - } - i.blockUpper = i.upper - if i.blockUpper != nil && i.cmp(i.blockUpper, i.index.Key().UserKey) > 0 { - // The upper-bound is greater than the index key which itself is greater - // than or equal to every key in the block. No need to check the - // upper-bound again for this block. - i.blockUpper = nil - } -} - type loadBlockResult int8 const ( @@ -489,2024 +85,6 @@ const ( loadBlockIrrelevant ) -// loadBlock loads the block at the current index position and leaves i.data -// unpositioned. If unsuccessful, it sets i.err to any error encountered, which -// may be nil if we have simply exhausted the entire table. -func (i *singleLevelIterator) loadBlock(dir int8) loadBlockResult { - if !i.index.valid() { - // Ensure the data block iterator is invalidated even if loading of the block - // fails. - i.data.invalidate() - return loadBlockFailed - } - // Load the next block. - v := i.index.value() - bhp, err := decodeBlockHandleWithProperties(v.InPlaceValue()) - if i.dataBH == bhp.BlockHandle && i.data.valid() { - // We're already at the data block we want to load. Reset bounds in case - // they changed since the last seek, but don't reload the block from cache - // or disk. - // - // It's safe to leave i.data in its original state here, as all callers to - // loadBlock make an absolute positioning call (i.e. a seek, first, or last) - // to `i.data` right after loadBlock returns loadBlockOK. - i.initBounds() - return loadBlockOK - } - // Ensure the data block iterator is invalidated even if loading of the block - // fails. - i.data.invalidate() - i.dataBH = bhp.BlockHandle - if err != nil { - i.err = errCorruptIndexEntry - return loadBlockFailed - } - if i.bpfs != nil { - intersects, err := i.bpfs.intersects(bhp.Props) - if err != nil { - i.err = errCorruptIndexEntry - return loadBlockFailed - } - if intersects == blockMaybeExcluded { - intersects = i.resolveMaybeExcluded(dir) - } - if intersects == blockExcluded { - i.maybeFilteredKeysSingleLevel = true - return loadBlockIrrelevant - } - // blockIntersects - } - block, err := i.readBlockWithStats(i.dataBH, i.dataRH) - if err != nil { - i.err = err - return loadBlockFailed - } - i.err = i.data.initHandle(i.cmp, block, i.reader.Properties.GlobalSeqNum) - if i.err != nil { - // The block is partially loaded, and we don't want it to appear valid. - i.data.invalidate() - return loadBlockFailed - } - i.initBounds() - return loadBlockOK -} - -// readBlockForVBR implements the blockProviderWhenOpen interface for use by -// the valueBlockReader. -func (i *singleLevelIterator) readBlockForVBR( - ctx context.Context, h BlockHandle, stats *base.InternalIteratorStats, -) (cache.Handle, error) { - return i.reader.readBlock(ctx, h, nil, i.vbRH, stats) -} - -// resolveMaybeExcluded is invoked when the block-property filterer has found -// that a block is excluded according to its properties but only if its bounds -// fall within the filter's current bounds. This function consults the -// apprioriate bound, depending on the iteration direction, and returns either -// `blockIntersects` or `blockMaybeExcluded`. -func (i *singleLevelIterator) resolveMaybeExcluded(dir int8) intersectsResult { - // TODO(jackson): We could first try comparing to top-level index block's - // key, and if within bounds avoid per-data block key comparisons. - - // This iterator is configured with a bound-limited block property - // filter. The bpf determined this block could be excluded from - // iteration based on the property encoded in the block handle. - // However, we still need to determine if the block is wholly - // contained within the filter's key bounds. - // - // External guarantees ensure all the block's keys are ≥ the - // filter's lower bound during forward iteration, and that all the - // block's keys are < the filter's upper bound during backward - // iteration. We only need to determine if the opposite bound is - // also met. - // - // The index separator in index.Key() provides an inclusive - // upper-bound for the data block's keys, guaranteeing that all its - // keys are ≤ index.Key(). For forward iteration, this is all we - // need. - if dir > 0 { - // Forward iteration. - if i.bpfs.boundLimitedFilter.KeyIsWithinUpperBound(i.index.Key()) { - return blockExcluded - } - return blockIntersects - } - - // Reverse iteration. - // - // Because we're iterating in the reverse direction, we don't yet have - // enough context available to determine if the block is wholly contained - // within its bounds. This case arises only during backward iteration, - // because of the way the index is structured. - // - // Consider a bound-limited bpf limited to the bounds [b,d), loading the - // block with separator `c`. During reverse iteration, the guarantee that - // all the block's keys are < `d` is externally provided, but no guarantee - // is made on the bpf's lower bound. The separator `c` only provides an - // inclusive upper bound on the block's keys, indicating that the - // corresponding block handle points to a block containing only keys ≤ `c`. - // - // To establish a lower bound, we step the index backwards to read the - // previous block's separator, which provides an inclusive lower bound on - // the original block's keys. Afterwards, we step forward to restore our - // index position. - if peekKey, _ := i.index.Prev(); peekKey == nil { - // The original block points to the first block of this index block. If - // there's a two-level index, it could potentially provide a lower - // bound, but the code refactoring necessary to read it doesn't seem - // worth the payoff. We fall through to loading the block. - } else if i.bpfs.boundLimitedFilter.KeyIsWithinLowerBound(peekKey) { - // The lower-bound on the original block falls within the filter's - // bounds, and we can skip the block (after restoring our current index - // position). - _, _ = i.index.Next() - return blockExcluded - } - _, _ = i.index.Next() - return blockIntersects -} - -func (i *singleLevelIterator) readBlockWithStats( - bh BlockHandle, rh objstorage.ReadHandle, -) (cache.Handle, error) { - return i.reader.readBlock(i.ctx, bh, nil, rh, i.stats) -} - -func (i *singleLevelIterator) initBoundsForAlreadyLoadedBlock() { - if i.data.firstKey.UserKey == nil { - panic("initBoundsForAlreadyLoadedBlock must not be called on empty or corrupted block") - } - i.blockLower = i.lower - if i.blockLower != nil { - if i.data.firstKey.UserKey != nil && i.cmp(i.blockLower, i.data.firstKey.UserKey) < 0 { - // The lower-bound is less than the first key in the block. No need - // to check the lower-bound again for this block. - i.blockLower = nil - } - } - i.blockUpper = i.upper - if i.blockUpper != nil && i.cmp(i.blockUpper, i.index.Key().UserKey) > 0 { - // The upper-bound is greater than the index key which itself is greater - // than or equal to every key in the block. No need to check the - // upper-bound again for this block. - i.blockUpper = nil - } -} - -// The number of times to call Next/Prev in a block before giving up and seeking. -// The value of 4 is arbitrary. -// TODO(sumeer): experiment with dynamic adjustment based on the history of -// seeks for a particular iterator. -const numStepsBeforeSeek = 4 - -func (i *singleLevelIterator) trySeekGEUsingNextWithinBlock( - key []byte, -) (k *InternalKey, v base.LazyValue, done bool) { - k, v = i.data.Key(), i.data.value() - for j := 0; j < numStepsBeforeSeek; j++ { - curKeyCmp := i.cmp(k.UserKey, key) - if curKeyCmp >= 0 { - if i.blockUpper != nil && i.cmp(k.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{}, true - } - return k, v, true - } - k, v = i.data.Next() - if k == nil { - break - } - } - return k, v, false -} - -func (i *singleLevelIterator) trySeekLTUsingPrevWithinBlock( - key []byte, -) (k *InternalKey, v base.LazyValue, done bool) { - k, v = i.data.Key(), i.data.value() - for j := 0; j < numStepsBeforeSeek; j++ { - curKeyCmp := i.cmp(k.UserKey, key) - if curKeyCmp < 0 { - if i.blockLower != nil && i.cmp(k.UserKey, i.blockLower) < 0 { - i.exhaustedBounds = -1 - return nil, base.LazyValue{}, true - } - return k, v, true - } - k, v = i.data.Prev() - if k == nil { - break - } - } - return k, v, false -} - -func (i *singleLevelIterator) recordOffset() uint64 { - offset := i.dataBH.Offset - if i.data.valid() { - // - i.dataBH.Length/len(i.data.data) is the compression ratio. If - // uncompressed, this is 1. - // - i.data.nextOffset is the uncompressed position of the current record - // in the block. - // - i.dataBH.Offset is the offset of the block in the sstable before - // decompression. - offset += (uint64(i.data.nextOffset) * i.dataBH.Length) / uint64(len(i.data.data)) - } else { - // Last entry in the block must increment bytes iterated by the size of the block trailer - // and restart points. - offset += i.dataBH.Length + blockTrailerLen - } - return offset -} - -// SeekGE implements internalIterator.SeekGE, as documented in the pebble -// package. Note that SeekGE only checks the upper bound. It is up to the -// caller to ensure that key is greater than or equal to the lower bound. -func (i *singleLevelIterator) SeekGE( - key []byte, flags base.SeekGEFlags, -) (*InternalKey, base.LazyValue) { - if flags.TrySeekUsingNext() { - // The i.exhaustedBounds comparison indicates that the upper bound was - // reached. The i.data.isDataInvalidated() indicates that the sstable was - // exhausted. - if (i.exhaustedBounds == +1 || i.data.isDataInvalidated()) && i.err == nil { - // Already exhausted, so return nil. - return nil, base.LazyValue{} - } - if i.err != nil { - // The current iterator position cannot be used. - flags = flags.DisableTrySeekUsingNext() - } - // INVARIANT: flags.TrySeekUsingNext() => i.err == nil && - // !i.exhaustedBounds==+1 && !i.data.isDataInvalidated(). That is, - // data-exhausted and bounds-exhausted, as defined earlier, are both - // false. Ths makes it safe to clear out i.exhaustedBounds and i.err - // before calling into seekGEHelper. - } - - i.exhaustedBounds = 0 - i.err = nil // clear cached iteration error - boundsCmp := i.boundsCmp - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - i.positionedUsingLatestBounds = true - return i.seekGEHelper(key, boundsCmp, flags) -} - -// seekGEHelper contains the common functionality for SeekGE and SeekPrefixGE. -func (i *singleLevelIterator) seekGEHelper( - key []byte, boundsCmp int, flags base.SeekGEFlags, -) (*InternalKey, base.LazyValue) { - // Invariant: trySeekUsingNext => !i.data.isDataInvalidated() && i.exhaustedBounds != +1 - - // SeekGE performs various step-instead-of-seeking optimizations: eg enabled - // by trySeekUsingNext, or by monotonically increasing bounds (i.boundsCmp). - // Care must be taken to ensure that when performing these optimizations and - // the iterator becomes exhausted, i.maybeFilteredKeys is set appropriately. - // Consider a previous SeekGE that filtered keys from k until the current - // iterator position. - // - // If the previous SeekGE exhausted the iterator, it's possible keys greater - // than or equal to the current search key were filtered. We must not reuse - // the current iterator position without remembering the previous value of - // maybeFilteredKeys. - - var dontSeekWithinBlock bool - if !i.data.isDataInvalidated() && !i.index.isDataInvalidated() && i.data.valid() && i.index.valid() && - boundsCmp > 0 && i.cmp(key, i.index.Key().UserKey) <= 0 { - // Fast-path: The bounds have moved forward and this SeekGE is - // respecting the lower bound (guaranteed by Iterator). We know that - // the iterator must already be positioned within or just outside the - // previous bounds. Therefore it cannot be positioned at a block (or - // the position within that block) that is ahead of the seek position. - // However it can be positioned at an earlier block. This fast-path to - // use Next() on the block is only applied when we are already at the - // block that the slow-path (the else-clause) would load -- this is - // the motivation for the i.cmp(key, i.index.Key().UserKey) <= 0 - // predicate. - i.initBoundsForAlreadyLoadedBlock() - ikey, val, done := i.trySeekGEUsingNextWithinBlock(key) - if done { - return ikey, val - } - if ikey == nil { - // Done with this block. - dontSeekWithinBlock = true - } - } else { - // Cannot use bounds monotonicity. But may be able to optimize if - // caller claimed externally known invariant represented by - // flags.TrySeekUsingNext(). - if flags.TrySeekUsingNext() { - // seekPrefixGE or SeekGE has already ensured - // !i.data.isDataInvalidated() && i.exhaustedBounds != +1 - currKey := i.data.Key() - value := i.data.value() - less := i.cmp(currKey.UserKey, key) < 0 - // We could be more sophisticated and confirm that the seek - // position is within the current block before applying this - // optimization. But there may be some benefit even if it is in - // the next block, since we can avoid seeking i.index. - for j := 0; less && j < numStepsBeforeSeek; j++ { - currKey, value = i.Next() - if currKey == nil { - return nil, base.LazyValue{} - } - less = i.cmp(currKey.UserKey, key) < 0 - } - if !less { - if i.blockUpper != nil && i.cmp(currKey.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} - } - return currKey, value - } - } - - // Slow-path. - - // Since we're re-seeking the iterator, the previous value of - // maybeFilteredKeysSingleLevel is irrelevant. If we filter out blocks - // during seeking, loadBlock will set it to true. - i.maybeFilteredKeysSingleLevel = false - - var ikey *InternalKey - if ikey, _ = i.index.SeekGE(key, flags.DisableTrySeekUsingNext()); ikey == nil { - // The target key is greater than any key in the index block. - // Invalidate the block iterator so that a subsequent call to Prev() - // will return the last key in the table. - i.data.invalidate() - return nil, base.LazyValue{} - } - result := i.loadBlock(+1) - if result == loadBlockFailed { - return nil, base.LazyValue{} - } - if result == loadBlockIrrelevant { - // Enforce the upper bound here since don't want to bother moving - // to the next block if upper bound is already exceeded. Note that - // the next block starts with keys >= ikey.UserKey since even - // though this is the block separator, the same user key can span - // multiple blocks. Since upper is exclusive we use >= below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} - } - // Want to skip to the next block. - dontSeekWithinBlock = true - } - } - if !dontSeekWithinBlock { - if ikey, val := i.data.SeekGE(key, flags.DisableTrySeekUsingNext()); ikey != nil { - if i.blockUpper != nil && i.cmp(ikey.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} - } - return ikey, val - } - } - return i.skipForward() -} - -// SeekPrefixGE implements internalIterator.SeekPrefixGE, as documented in the -// pebble package. Note that SeekPrefixGE only checks the upper bound. It is up -// to the caller to ensure that key is greater than or equal to the lower bound. -func (i *singleLevelIterator) SeekPrefixGE( - prefix, key []byte, flags base.SeekGEFlags, -) (*base.InternalKey, base.LazyValue) { - k, v := i.seekPrefixGE(prefix, key, flags, i.useFilter) - return k, v -} - -func (i *singleLevelIterator) seekPrefixGE( - prefix, key []byte, flags base.SeekGEFlags, checkFilter bool, -) (k *InternalKey, value base.LazyValue) { - // NOTE: prefix is only used for bloom filter checking and not later work in - // this method. Hence, we can use the existing iterator position if the last - // SeekPrefixGE did not fail bloom filter matching. - - err := i.err - i.err = nil // clear cached iteration error - if checkFilter && i.reader.tableFilter != nil { - if !i.lastBloomFilterMatched { - // Iterator is not positioned based on last seek. - flags = flags.DisableTrySeekUsingNext() - } - i.lastBloomFilterMatched = false - // Check prefix bloom filter. - var dataH cache.Handle - dataH, i.err = i.reader.readFilter(i.ctx, i.stats) - if i.err != nil { - i.data.invalidate() - return nil, base.LazyValue{} - } - mayContain := i.reader.tableFilter.mayContain(dataH.Get(), prefix) - dataH.Release() - if !mayContain { - // This invalidation may not be necessary for correctness, and may - // be a place to optimize later by reusing the already loaded - // block. It was necessary in earlier versions of the code since - // the caller was allowed to call Next when SeekPrefixGE returned - // nil. This is no longer allowed. - i.data.invalidate() - return nil, base.LazyValue{} - } - i.lastBloomFilterMatched = true - } - if flags.TrySeekUsingNext() { - // The i.exhaustedBounds comparison indicates that the upper bound was - // reached. The i.data.isDataInvalidated() indicates that the sstable was - // exhausted. - if (i.exhaustedBounds == +1 || i.data.isDataInvalidated()) && err == nil { - // Already exhausted, so return nil. - return nil, base.LazyValue{} - } - if err != nil { - // The current iterator position cannot be used. - flags = flags.DisableTrySeekUsingNext() - } - // INVARIANT: flags.TrySeekUsingNext() => err == nil && - // !i.exhaustedBounds==+1 && !i.data.isDataInvalidated(). That is, - // data-exhausted and bounds-exhausted, as defined earlier, are both - // false. Ths makes it safe to clear out i.exhaustedBounds and i.err - // before calling into seekGEHelper. - } - // Bloom filter matches, or skipped, so this method will position the - // iterator. - i.exhaustedBounds = 0 - boundsCmp := i.boundsCmp - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - i.positionedUsingLatestBounds = true - k, value = i.seekGEHelper(key, boundsCmp, flags) - return k, value -} - -// SeekLT implements internalIterator.SeekLT, as documented in the pebble -// package. Note that SeekLT only checks the lower bound. It is up to the -// caller to ensure that key is less than the upper bound. -func (i *singleLevelIterator) SeekLT( - key []byte, flags base.SeekLTFlags, -) (*InternalKey, base.LazyValue) { - i.exhaustedBounds = 0 - i.err = nil // clear cached iteration error - boundsCmp := i.boundsCmp - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - - // Seeking operations perform various step-instead-of-seeking optimizations: - // eg by considering monotonically increasing bounds (i.boundsCmp). Care - // must be taken to ensure that when performing these optimizations and the - // iterator becomes exhausted i.maybeFilteredKeysSingleLevel is set - // appropriately. Consider a previous SeekLT that filtered keys from k - // until the current iterator position. - // - // If the previous SeekLT did exhausted the iterator, it's possible keys - // less than the current search key were filtered. We must not reuse the - // current iterator position without remembering the previous value of - // maybeFilteredKeysSingleLevel. - - i.positionedUsingLatestBounds = true - - var dontSeekWithinBlock bool - if !i.data.isDataInvalidated() && !i.index.isDataInvalidated() && i.data.valid() && i.index.valid() && - boundsCmp < 0 && i.cmp(i.data.firstKey.UserKey, key) < 0 { - // Fast-path: The bounds have moved backward, and this SeekLT is - // respecting the upper bound (guaranteed by Iterator). We know that - // the iterator must already be positioned within or just outside the - // previous bounds. Therefore it cannot be positioned at a block (or - // the position within that block) that is behind the seek position. - // However it can be positioned at a later block. This fast-path to - // use Prev() on the block is only applied when we are already at the - // block that can satisfy this seek -- this is the motivation for the - // the i.cmp(i.data.firstKey.UserKey, key) < 0 predicate. - i.initBoundsForAlreadyLoadedBlock() - ikey, val, done := i.trySeekLTUsingPrevWithinBlock(key) - if done { - return ikey, val - } - if ikey == nil { - // Done with this block. - dontSeekWithinBlock = true - } - } else { - // Slow-path. - i.maybeFilteredKeysSingleLevel = false - var ikey *InternalKey - - // NB: If a bound-limited block property filter is configured, it's - // externally ensured that the filter is disabled (through returning - // Intersects=false irrespective of the block props provided) during - // seeks. - if ikey, _ = i.index.SeekGE(key, base.SeekGEFlagsNone); ikey == nil { - ikey, _ = i.index.Last() - if ikey == nil { - return nil, base.LazyValue{} - } - } - // INVARIANT: ikey != nil. - result := i.loadBlock(-1) - if result == loadBlockFailed { - return nil, base.LazyValue{} - } - if result == loadBlockIrrelevant { - // Enforce the lower bound here since don't want to bother moving - // to the previous block if lower bound is already exceeded. Note - // that the previous block starts with keys <= ikey.UserKey since - // even though this is the current block's separator, the same - // user key can span multiple blocks. - if i.lower != nil && i.cmp(ikey.UserKey, i.lower) < 0 { - i.exhaustedBounds = -1 - return nil, base.LazyValue{} - } - // Want to skip to the previous block. - dontSeekWithinBlock = true - } - } - if !dontSeekWithinBlock { - if ikey, val := i.data.SeekLT(key, flags); ikey != nil { - if i.blockLower != nil && i.cmp(ikey.UserKey, i.blockLower) < 0 { - i.exhaustedBounds = -1 - return nil, base.LazyValue{} - } - return ikey, val - } - } - // The index contains separator keys which may lie between - // user-keys. Consider the user-keys: - // - // complete - // ---- new block --- - // complexion - // - // If these two keys end one block and start the next, the index key may - // be chosen as "compleu". The SeekGE in the index block will then point - // us to the block containing "complexion". If this happens, we want the - // last key from the previous data block. - return i.skipBackward() -} - -// First implements internalIterator.First, as documented in the pebble -// package. Note that First only checks the upper bound. It is up to the caller -// to ensure that key is greater than or equal to the lower bound (e.g. via a -// call to SeekGE(lower)). -func (i *singleLevelIterator) First() (*InternalKey, base.LazyValue) { - if i.lower != nil { - panic("singleLevelIterator.First() used despite lower bound") - } - i.positionedUsingLatestBounds = true - i.maybeFilteredKeysSingleLevel = false - return i.firstInternal() -} - -// firstInternal is a helper used for absolute positioning in a single-level -// index file, or for positioning in the second-level index in a two-level -// index file. For the latter, one cannot make any claims about absolute -// positioning. -func (i *singleLevelIterator) firstInternal() (*InternalKey, base.LazyValue) { - i.exhaustedBounds = 0 - i.err = nil // clear cached iteration error - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - - var ikey *InternalKey - if ikey, _ = i.index.First(); ikey == nil { - i.data.invalidate() - return nil, base.LazyValue{} - } - result := i.loadBlock(+1) - if result == loadBlockFailed { - return nil, base.LazyValue{} - } - if result == loadBlockOK { - if ikey, val := i.data.First(); ikey != nil { - if i.blockUpper != nil && i.cmp(ikey.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} - } - return ikey, val - } - // Else fall through to skipForward. - } else { - // result == loadBlockIrrelevant. Enforce the upper bound here since - // don't want to bother moving to the next block if upper bound is - // already exceeded. Note that the next block starts with keys >= - // ikey.UserKey since even though this is the block separator, the - // same user key can span multiple blocks. Since upper is exclusive we - // use >= below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} - } - // Else fall through to skipForward. - } - - return i.skipForward() -} - -// Last implements internalIterator.Last, as documented in the pebble -// package. Note that Last only checks the lower bound. It is up to the caller -// to ensure that key is less than the upper bound (e.g. via a call to -// SeekLT(upper)) -func (i *singleLevelIterator) Last() (*InternalKey, base.LazyValue) { - if i.upper != nil { - panic("singleLevelIterator.Last() used despite upper bound") - } - i.positionedUsingLatestBounds = true - i.maybeFilteredKeysSingleLevel = false - return i.lastInternal() -} - -// lastInternal is a helper used for absolute positioning in a single-level -// index file, or for positioning in the second-level index in a two-level -// index file. For the latter, one cannot make any claims about absolute -// positioning. -func (i *singleLevelIterator) lastInternal() (*InternalKey, base.LazyValue) { - i.exhaustedBounds = 0 - i.err = nil // clear cached iteration error - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - - var ikey *InternalKey - if ikey, _ = i.index.Last(); ikey == nil { - i.data.invalidate() - return nil, base.LazyValue{} - } - result := i.loadBlock(-1) - if result == loadBlockFailed { - return nil, base.LazyValue{} - } - if result == loadBlockOK { - if ikey, val := i.data.Last(); ikey != nil { - if i.blockLower != nil && i.cmp(ikey.UserKey, i.blockLower) < 0 { - i.exhaustedBounds = -1 - return nil, base.LazyValue{} - } - return ikey, val - } - // Else fall through to skipBackward. - } else { - // result == loadBlockIrrelevant. Enforce the lower bound here since - // don't want to bother moving to the previous block if lower bound is - // already exceeded. Note that the previous block starts with keys <= - // key.UserKey since even though this is the current block's - // separator, the same user key can span multiple blocks. - if i.lower != nil && i.cmp(ikey.UserKey, i.lower) < 0 { - i.exhaustedBounds = -1 - return nil, base.LazyValue{} - } - } - - return i.skipBackward() -} - -// Next implements internalIterator.Next, as documented in the pebble -// package. -// Note: compactionIterator.Next mirrors the implementation of Iterator.Next -// due to performance. Keep the two in sync. -func (i *singleLevelIterator) Next() (*InternalKey, base.LazyValue) { - if i.exhaustedBounds == +1 { - panic("Next called even though exhausted upper bound") - } - i.exhaustedBounds = 0 - i.maybeFilteredKeysSingleLevel = false - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - - if i.err != nil { - return nil, base.LazyValue{} - } - if key, val := i.data.Next(); key != nil { - if i.blockUpper != nil && i.cmp(key.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} - } - return key, val - } - return i.skipForward() -} - -// NextPrefix implements (base.InternalIterator).NextPrefix. -func (i *singleLevelIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { - if i.exhaustedBounds == +1 { - panic("NextPrefix called even though exhausted upper bound") - } - i.exhaustedBounds = 0 - i.maybeFilteredKeysSingleLevel = false - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - if i.err != nil { - return nil, base.LazyValue{} - } - if key, val := i.data.nextPrefix(succKey); key != nil { - if i.blockUpper != nil && i.cmp(key.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} - } - return key, val - } - // Did not find prefix in the existing data block. This is the slow-path - // where we effectively seek the iterator. - var ikey *InternalKey - // The key is likely to be in the next data block, so try one step. - if ikey, _ = i.index.Next(); ikey == nil { - // The target key is greater than any key in the index block. - // Invalidate the block iterator so that a subsequent call to Prev() - // will return the last key in the table. - i.data.invalidate() - return nil, base.LazyValue{} - } - if i.cmp(succKey, ikey.UserKey) > 0 { - // Not in the next data block, so seek the index. - if ikey, _ = i.index.SeekGE(succKey, base.SeekGEFlagsNone); ikey == nil { - // The target key is greater than any key in the index block. - // Invalidate the block iterator so that a subsequent call to Prev() - // will return the last key in the table. - i.data.invalidate() - return nil, base.LazyValue{} - } - } - result := i.loadBlock(+1) - if result == loadBlockFailed { - return nil, base.LazyValue{} - } - if result == loadBlockIrrelevant { - // Enforce the upper bound here since don't want to bother moving - // to the next block if upper bound is already exceeded. Note that - // the next block starts with keys >= ikey.UserKey since even - // though this is the block separator, the same user key can span - // multiple blocks. Since upper is exclusive we use >= below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} - } - } else if key, val := i.data.SeekGE(succKey, base.SeekGEFlagsNone); key != nil { - if i.blockUpper != nil && i.cmp(key.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} - } - return key, val - } - return i.skipForward() -} - -// Prev implements internalIterator.Prev, as documented in the pebble -// package. -func (i *singleLevelIterator) Prev() (*InternalKey, base.LazyValue) { - if i.exhaustedBounds == -1 { - panic("Prev called even though exhausted lower bound") - } - i.exhaustedBounds = 0 - i.maybeFilteredKeysSingleLevel = false - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - - if i.err != nil { - return nil, base.LazyValue{} - } - if key, val := i.data.Prev(); key != nil { - if i.blockLower != nil && i.cmp(key.UserKey, i.blockLower) < 0 { - i.exhaustedBounds = -1 - return nil, base.LazyValue{} - } - return key, val - } - return i.skipBackward() -} - -func (i *singleLevelIterator) skipForward() (*InternalKey, base.LazyValue) { - for { - var key *InternalKey - if key, _ = i.index.Next(); key == nil { - i.data.invalidate() - break - } - result := i.loadBlock(+1) - if result != loadBlockOK { - if i.err != nil { - break - } - if result == loadBlockFailed { - // We checked that i.index was at a valid entry, so - // loadBlockFailed could not have happened due to to i.index - // being exhausted, and must be due to an error. - panic("loadBlock should not have failed with no error") - } - // result == loadBlockIrrelevant. Enforce the upper bound here - // since don't want to bother moving to the next block if upper - // bound is already exceeded. Note that the next block starts with - // keys >= key.UserKey since even though this is the block - // separator, the same user key can span multiple blocks. Since - // upper is exclusive we use >= below. - if i.upper != nil && i.cmp(key.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} - } - continue - } - if key, val := i.data.First(); key != nil { - if i.blockUpper != nil && i.cmp(key.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} - } - return key, val - } - } - return nil, base.LazyValue{} -} - -func (i *singleLevelIterator) skipBackward() (*InternalKey, base.LazyValue) { - for { - var key *InternalKey - if key, _ = i.index.Prev(); key == nil { - i.data.invalidate() - break - } - result := i.loadBlock(-1) - if result != loadBlockOK { - if i.err != nil { - break - } - if result == loadBlockFailed { - // We checked that i.index was at a valid entry, so - // loadBlockFailed could not have happened due to to i.index - // being exhausted, and must be due to an error. - panic("loadBlock should not have failed with no error") - } - // result == loadBlockIrrelevant. Enforce the lower bound here - // since don't want to bother moving to the previous block if lower - // bound is already exceeded. Note that the previous block starts with - // keys <= key.UserKey since even though this is the current block's - // separator, the same user key can span multiple blocks. - if i.lower != nil && i.cmp(key.UserKey, i.lower) < 0 { - i.exhaustedBounds = -1 - return nil, base.LazyValue{} - } - continue - } - key, val := i.data.Last() - if key == nil { - return nil, base.LazyValue{} - } - if i.blockLower != nil && i.cmp(key.UserKey, i.blockLower) < 0 { - i.exhaustedBounds = -1 - return nil, base.LazyValue{} - } - return key, val - } - return nil, base.LazyValue{} -} - -// Error implements internalIterator.Error, as documented in the pebble -// package. -func (i *singleLevelIterator) Error() error { - if err := i.data.Error(); err != nil { - return err - } - return i.err -} - -// MaybeFilteredKeys may be called when an iterator is exhausted to indicate -// whether or not the last positioning method may have skipped any keys due to -// block-property filters. -func (i *singleLevelIterator) MaybeFilteredKeys() bool { - return i.maybeFilteredKeysSingleLevel -} - -// SetCloseHook sets a function that will be called when the iterator is -// closed. -func (i *singleLevelIterator) SetCloseHook(fn func(i Iterator) error) { - i.closeHook = fn -} - -func firstError(err0, err1 error) error { - if err0 != nil { - return err0 - } - return err1 -} - -// Close implements internalIterator.Close, as documented in the pebble -// package. -func (i *singleLevelIterator) Close() error { - var err error - if i.closeHook != nil { - err = firstError(err, i.closeHook(i)) - } - err = firstError(err, i.data.Close()) - err = firstError(err, i.index.Close()) - if i.dataRH != nil { - err = firstError(err, i.dataRH.Close()) - i.dataRH = nil - } - err = firstError(err, i.err) - if i.bpfs != nil { - releaseBlockPropertiesFilterer(i.bpfs) - } - if i.vbReader != nil { - i.vbReader.close() - } - if i.vbRH != nil { - err = firstError(err, i.vbRH.Close()) - i.vbRH = nil - } - *i = i.resetForReuse() - singleLevelIterPool.Put(i) - return err -} - -func (i *singleLevelIterator) String() string { - return i.reader.fileNum.String() -} - -// Deterministic disabling of the bounds-based optimization that avoids seeking. -// Uses the iterator pointer, since we want diversity in iterator behavior for -// the same SetBounds call. Used for tests. -func disableBoundsOpt(bound []byte, ptr uintptr) bool { - // Fibonacci hash https://probablydance.com/2018/06/16/fibonacci-hashing-the-optimization-that-the-world-forgot-or-a-better-alternative-to-integer-modulo/ - simpleHash := (11400714819323198485 * uint64(ptr)) >> 63 - return bound[len(bound)-1]&byte(1) == 0 && simpleHash == 0 -} - -// ensureBoundsOptDeterminism provides a facility for disabling of the bounds -// optimizations performed by disableBoundsOpt for tests that require -// deterministic iterator behavior. Some unit tests examine internal iterator -// state and require this behavior to be deterministic. -var ensureBoundsOptDeterminism bool - -// SetBounds implements internalIterator.SetBounds, as documented in the pebble -// package. -func (i *singleLevelIterator) SetBounds(lower, upper []byte) { - i.boundsCmp = 0 - if i.positionedUsingLatestBounds { - if i.upper != nil && lower != nil && i.cmp(i.upper, lower) <= 0 { - i.boundsCmp = +1 - if invariants.Enabled && !ensureBoundsOptDeterminism && - disableBoundsOpt(lower, uintptr(unsafe.Pointer(i))) { - i.boundsCmp = 0 - } - } else if i.lower != nil && upper != nil && i.cmp(upper, i.lower) <= 0 { - i.boundsCmp = -1 - if invariants.Enabled && !ensureBoundsOptDeterminism && - disableBoundsOpt(upper, uintptr(unsafe.Pointer(i))) { - i.boundsCmp = 0 - } - } - i.positionedUsingLatestBounds = false - } - i.lower = lower - i.upper = upper - i.blockLower = nil - i.blockUpper = nil -} - -var _ base.InternalIterator = &singleLevelIterator{} -var _ base.InternalIterator = &twoLevelIterator{} - -// compactionIterator is similar to Iterator but it increments the number of -// bytes that have been iterated through. -type compactionIterator struct { - *singleLevelIterator - bytesIterated *uint64 - prevOffset uint64 -} - -// compactionIterator implements the base.InternalIterator interface. -var _ base.InternalIterator = (*compactionIterator)(nil) - -func (i *compactionIterator) String() string { - return i.reader.fileNum.String() -} - -func (i *compactionIterator) SeekGE( - key []byte, flags base.SeekGEFlags, -) (*InternalKey, base.LazyValue) { - panic("pebble: SeekGE unimplemented") -} - -func (i *compactionIterator) SeekPrefixGE( - prefix, key []byte, flags base.SeekGEFlags, -) (*base.InternalKey, base.LazyValue) { - panic("pebble: SeekPrefixGE unimplemented") -} - -func (i *compactionIterator) SeekLT( - key []byte, flags base.SeekLTFlags, -) (*InternalKey, base.LazyValue) { - panic("pebble: SeekLT unimplemented") -} - -func (i *compactionIterator) First() (*InternalKey, base.LazyValue) { - i.err = nil // clear cached iteration error - return i.skipForward(i.singleLevelIterator.First()) -} - -func (i *compactionIterator) Last() (*InternalKey, base.LazyValue) { - panic("pebble: Last unimplemented") -} - -// Note: compactionIterator.Next mirrors the implementation of Iterator.Next -// due to performance. Keep the two in sync. -func (i *compactionIterator) Next() (*InternalKey, base.LazyValue) { - if i.err != nil { - return nil, base.LazyValue{} - } - return i.skipForward(i.data.Next()) -} - -func (i *compactionIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { - panic("pebble: NextPrefix unimplemented") -} - -func (i *compactionIterator) Prev() (*InternalKey, base.LazyValue) { - panic("pebble: Prev unimplemented") -} - -func (i *compactionIterator) skipForward( - key *InternalKey, val base.LazyValue, -) (*InternalKey, base.LazyValue) { - if key == nil { - for { - if key, _ := i.index.Next(); key == nil { - break - } - result := i.loadBlock(+1) - if result != loadBlockOK { - if i.err != nil { - break - } - switch result { - case loadBlockFailed: - // We checked that i.index was at a valid entry, so - // loadBlockFailed could not have happened due to to i.index - // being exhausted, and must be due to an error. - panic("loadBlock should not have failed with no error") - case loadBlockIrrelevant: - panic("compactionIter should not be using block intervals for skipping") - default: - panic(fmt.Sprintf("unexpected case %d", result)) - } - } - // result == loadBlockOK - if key, val = i.data.First(); key != nil { - break - } - } - } - - curOffset := i.recordOffset() - *i.bytesIterated += uint64(curOffset - i.prevOffset) - i.prevOffset = curOffset - return key, val -} - -type twoLevelIterator struct { - singleLevelIterator - // maybeFilteredKeysSingleLevel indicates whether the last iterator - // positioning operation may have skipped any index blocks due to - // block-property filters when positioning the top-level-index. - maybeFilteredKeysTwoLevel bool - topLevelIndex blockIter -} - -// twoLevelIterator implements the base.InternalIterator interface. -var _ base.InternalIterator = (*twoLevelIterator)(nil) - -// loadIndex loads the index block at the current top level index position and -// leaves i.index unpositioned. If unsuccessful, it gets i.err to any error -// encountered, which may be nil if we have simply exhausted the entire table. -// This is used for two level indexes. -func (i *twoLevelIterator) loadIndex(dir int8) loadBlockResult { - // Ensure the index data block iterators are invalidated even if loading of - // the index fails. - i.data.invalidate() - i.index.invalidate() - if !i.topLevelIndex.valid() { - i.index.offset = 0 - i.index.restarts = 0 - return loadBlockFailed - } - v := i.topLevelIndex.value() - bhp, err := decodeBlockHandleWithProperties(v.InPlaceValue()) - if err != nil { - i.err = base.CorruptionErrorf("pebble/table: corrupt top level index entry") - return loadBlockFailed - } - if i.bpfs != nil { - intersects, err := i.bpfs.intersects(bhp.Props) - if err != nil { - i.err = errCorruptIndexEntry - return loadBlockFailed - } - if intersects == blockMaybeExcluded { - intersects = i.resolveMaybeExcluded(dir) - } - if intersects == blockExcluded { - i.maybeFilteredKeysTwoLevel = true - return loadBlockIrrelevant - } - // blockIntersects - } - indexBlock, err := i.readBlockWithStats(bhp.BlockHandle, nil /* readHandle */) - if err != nil { - i.err = err - return loadBlockFailed - } - if i.err = i.index.initHandle( - i.cmp, indexBlock, i.reader.Properties.GlobalSeqNum); i.err == nil { - return loadBlockOK - } - return loadBlockFailed -} - -// resolveMaybeExcluded is invoked when the block-property filterer has found -// that an index block is excluded according to its properties but only if its -// bounds fall within the filter's current bounds. This function consults the -// apprioriate bound, depending on the iteration direction, and returns either -// `blockIntersects` or -// `blockMaybeExcluded`. -func (i *twoLevelIterator) resolveMaybeExcluded(dir int8) intersectsResult { - // This iterator is configured with a bound-limited block property filter. - // The bpf determined this entire index block could be excluded from - // iteration based on the property encoded in the block handle. However, we - // still need to determine if the index block is wholly contained within the - // filter's key bounds. - // - // External guarantees ensure all its data blocks' keys are ≥ the filter's - // lower bound during forward iteration, and that all its data blocks' keys - // are < the filter's upper bound during backward iteration. We only need to - // determine if the opposite bound is also met. - // - // The index separator in topLevelIndex.Key() provides an inclusive - // upper-bound for the index block's keys, guaranteeing that all its keys - // are ≤ topLevelIndex.Key(). For forward iteration, this is all we need. - if dir > 0 { - // Forward iteration. - if i.bpfs.boundLimitedFilter.KeyIsWithinUpperBound(i.topLevelIndex.Key()) { - return blockExcluded - } - return blockIntersects - } - - // Reverse iteration. - // - // Because we're iterating in the reverse direction, we don't yet have - // enough context available to determine if the block is wholly contained - // within its bounds. This case arises only during backward iteration, - // because of the way the index is structured. - // - // Consider a bound-limited bpf limited to the bounds [b,d), loading the - // block with separator `c`. During reverse iteration, the guarantee that - // all the block's keys are < `d` is externally provided, but no guarantee - // is made on the bpf's lower bound. The separator `c` only provides an - // inclusive upper bound on the block's keys, indicating that the - // corresponding block handle points to a block containing only keys ≤ `c`. - // - // To establish a lower bound, we step the top-level index backwards to read - // the previous block's separator, which provides an inclusive lower bound - // on the original index block's keys. Afterwards, we step forward to - // restore our top-level index position. - if peekKey, _ := i.topLevelIndex.Prev(); peekKey == nil { - // The original block points to the first index block of this table. If - // we knew the lower bound for the entire table, it could provide a - // lower bound, but the code refactoring necessary to read it doesn't - // seem worth the payoff. We fall through to loading the block. - } else if i.bpfs.boundLimitedFilter.KeyIsWithinLowerBound(peekKey) { - // The lower-bound on the original index block falls within the filter's - // bounds, and we can skip the block (after restoring our current - // top-level index position). - _, _ = i.topLevelIndex.Next() - return blockExcluded - } - _, _ = i.topLevelIndex.Next() - return blockIntersects -} - -func (i *twoLevelIterator) init( - ctx context.Context, - r *Reader, - lower, upper []byte, - filterer *BlockPropertiesFilterer, - useFilter bool, - stats *base.InternalIteratorStats, - rp ReaderProvider, -) error { - if r.err != nil { - return r.err - } - topLevelIndexH, err := r.readIndex(ctx, stats) - if err != nil { - return err - } - - i.ctx = ctx - i.lower = lower - i.upper = upper - i.bpfs = filterer - i.useFilter = useFilter - i.reader = r - i.cmp = r.Compare - i.stats = stats - err = i.topLevelIndex.initHandle(i.cmp, topLevelIndexH, r.Properties.GlobalSeqNum) - if err != nil { - // blockIter.Close releases topLevelIndexH and always returns a nil error - _ = i.topLevelIndex.Close() - return err - } - i.dataRH = r.readable.NewReadHandle() - if r.tableFormat == TableFormatPebblev3 { - if r.Properties.NumValueBlocks > 0 { - i.vbReader = &valueBlockReader{ - ctx: ctx, - bpOpen: i, - rp: rp, - vbih: r.valueBIH, - stats: stats, - } - i.data.lazyValueHandling.vbr = i.vbReader - i.vbRH = r.readable.NewReadHandle() - } - i.data.lazyValueHandling.hasValuePrefix = true - } - return nil -} - -func (i *twoLevelIterator) String() string { - return i.reader.fileNum.String() -} - -// MaybeFilteredKeys may be called when an iterator is exhausted to indicate -// whether or not the last positioning method may have skipped any keys due to -// block-property filters. -func (i *twoLevelIterator) MaybeFilteredKeys() bool { - // While reading sstables with two-level indexes, knowledge of whether we've - // filtered keys is tracked separately for each index level. The - // seek-using-next optimizations have different criteria. We can only reset - // maybeFilteredKeys back to false during a seek when NOT using the - // fast-path that uses the current iterator position. - // - // If either level might have filtered keys to arrive at the current - // iterator position, return MaybeFilteredKeys=true. - return i.maybeFilteredKeysTwoLevel || i.maybeFilteredKeysSingleLevel -} - -// SeekGE implements internalIterator.SeekGE, as documented in the pebble -// package. Note that SeekGE only checks the upper bound. It is up to the -// caller to ensure that key is greater than or equal to the lower bound. -func (i *twoLevelIterator) SeekGE( - key []byte, flags base.SeekGEFlags, -) (*InternalKey, base.LazyValue) { - err := i.err - i.err = nil // clear cached iteration error - - // The twoLevelIterator could be already exhausted. Utilize that when - // trySeekUsingNext is true. See the comment about data-exhausted, PGDE, and - // bounds-exhausted near the top of the file. - if flags.TrySeekUsingNext() && - (i.exhaustedBounds == +1 || (i.data.isDataInvalidated() && i.index.isDataInvalidated())) && - err == nil { - // Already exhausted, so return nil. - return nil, base.LazyValue{} - } - - // SeekGE performs various step-instead-of-seeking optimizations: eg enabled - // by trySeekUsingNext, or by monotonically increasing bounds (i.boundsCmp). - // Care must be taken to ensure that when performing these optimizations and - // the iterator becomes exhausted, i.maybeFilteredKeys is set appropriately. - // Consider a previous SeekGE that filtered keys from k until the current - // iterator position. - // - // If the previous SeekGE exhausted the iterator while seeking within the - // two-level index, it's possible keys greater than or equal to the current - // search key were filtered through skipped index blocks. We must not reuse - // the position of the two-level index iterator without remembering the - // previous value of maybeFilteredKeys. - - // We fall into the slow path if i.index.isDataInvalidated() even if the - // top-level iterator is already positioned correctly and all other - // conditions are met. An alternative structure could reuse topLevelIndex's - // current position and reload the index block to which it points. Arguably, - // an index block load is expensive and the index block may still be earlier - // than the index block containing the sought key, resulting in a wasteful - // block load. - - var dontSeekWithinSingleLevelIter bool - if i.topLevelIndex.isDataInvalidated() || !i.topLevelIndex.valid() || i.index.isDataInvalidated() || err != nil || - (i.boundsCmp <= 0 && !flags.TrySeekUsingNext()) || i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 { - // Slow-path: need to position the topLevelIndex. - - // The previous exhausted state of singleLevelIterator is no longer - // relevant, since we may be moving to a different index block. - i.exhaustedBounds = 0 - i.maybeFilteredKeysTwoLevel = false - flags = flags.DisableTrySeekUsingNext() - var ikey *InternalKey - if ikey, _ = i.topLevelIndex.SeekGE(key, flags); ikey == nil { - i.data.invalidate() - i.index.invalidate() - return nil, base.LazyValue{} - } - - result := i.loadIndex(+1) - if result == loadBlockFailed { - i.boundsCmp = 0 - return nil, base.LazyValue{} - } - if result == loadBlockIrrelevant { - // Enforce the upper bound here since don't want to bother moving - // to the next entry in the top level index if upper bound is - // already exceeded. Note that the next entry starts with keys >= - // ikey.UserKey since even though this is the block separator, the - // same user key can span multiple index blocks. Since upper is - // exclusive we use >= below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 - } - // Fall through to skipForward. - dontSeekWithinSingleLevelIter = true - // Clear boundsCmp. - // - // In the typical cases where dontSeekWithinSingleLevelIter=false, - // the singleLevelIterator.SeekGE call will clear boundsCmp. - // However, in this case where dontSeekWithinSingleLevelIter=true, - // we never seek on the single-level iterator. This call will fall - // through to skipForward, which may improperly leave boundsCmp=+1 - // unless we clear it here. - i.boundsCmp = 0 - } - } else { - // INVARIANT: err == nil. - // - // Else fast-path: There are two possible cases, from - // (i.boundsCmp > 0 || flags.TrySeekUsingNext()): - // - // 1) The bounds have moved forward (i.boundsCmp > 0) and this SeekGE is - // respecting the lower bound (guaranteed by Iterator). We know that the - // iterator must already be positioned within or just outside the previous - // bounds. Therefore, the topLevelIndex iter cannot be positioned at an - // entry ahead of the seek position (though it can be positioned behind). - // The !i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 confirms that it is - // not behind. Since it is not ahead and not behind it must be at the - // right position. - // - // 2) This SeekGE will land on a key that is greater than the key we are - // currently at (guaranteed by trySeekUsingNext), but since i.cmp(key, - // i.topLevelIndex.Key().UserKey) <= 0, we are at the correct lower level - // index block. No need to reset the state of singleLevelIterator. - // - // Note that cases 1 and 2 never overlap, and one of them must be true, - // but we have some test code (TestIterRandomizedMaybeFilteredKeys) that - // sets both to true, so we fix things here and then do an invariant - // check. - // - // This invariant checking is important enough that we do not gate it - // behind invariants.Enabled. - if i.boundsCmp > 0 { - // TODO(sumeer): fix TestIterRandomizedMaybeFilteredKeys so as to not - // need this behavior. - flags = flags.DisableTrySeekUsingNext() - } - if i.boundsCmp > 0 == flags.TrySeekUsingNext() { - panic(fmt.Sprintf("inconsistency in optimization case 1 %t and case 2 %t", - i.boundsCmp > 0, flags.TrySeekUsingNext())) - } - - if !flags.TrySeekUsingNext() { - // Case 1. Bounds have changed so the previous exhausted bounds state is - // irrelevant. - // WARNING-data-exhausted: this is safe to do only because the monotonic - // bounds optimizations only work when !data-exhausted. If they also - // worked with data-exhausted, we have made it unclear whether - // data-exhausted is actually true. See the comment at the top of the - // file. - i.exhaustedBounds = 0 - } - // Else flags.TrySeekUsingNext(). The i.exhaustedBounds is important to - // preserve for singleLevelIterator, and twoLevelIterator.skipForward. See - // bug https://github.com/cockroachdb/pebble/issues/2036. - } - - if !dontSeekWithinSingleLevelIter { - // Note that while trySeekUsingNext could be false here, singleLevelIterator - // could do its own boundsCmp-based optimization to seek using next. - if ikey, val := i.singleLevelIterator.SeekGE(key, flags); ikey != nil { - return ikey, val - } - } - return i.skipForward() -} - -// SeekPrefixGE implements internalIterator.SeekPrefixGE, as documented in the -// pebble package. Note that SeekPrefixGE only checks the upper bound. It is up -// to the caller to ensure that key is greater than or equal to the lower bound. -func (i *twoLevelIterator) SeekPrefixGE( - prefix, key []byte, flags base.SeekGEFlags, -) (*base.InternalKey, base.LazyValue) { - // NOTE: prefix is only used for bloom filter checking and not later work in - // this method. Hence, we can use the existing iterator position if the last - // SeekPrefixGE did not fail bloom filter matching. - - err := i.err - i.err = nil // clear cached iteration error - - // The twoLevelIterator could be already exhausted. Utilize that when - // trySeekUsingNext is true. See the comment about data-exhausted, PGDE, and - // bounds-exhausted near the top of the file. - filterUsedAndDidNotMatch := - i.reader.tableFilter != nil && i.useFilter && !i.lastBloomFilterMatched - if flags.TrySeekUsingNext() && !filterUsedAndDidNotMatch && - (i.exhaustedBounds == +1 || (i.data.isDataInvalidated() && i.index.isDataInvalidated())) && - err == nil { - // Already exhausted, so return nil. - return nil, base.LazyValue{} - } - - // Check prefix bloom filter. - if i.reader.tableFilter != nil && i.useFilter { - if !i.lastBloomFilterMatched { - // Iterator is not positioned based on last seek. - flags = flags.DisableTrySeekUsingNext() - } - i.lastBloomFilterMatched = false - var dataH cache.Handle - dataH, i.err = i.reader.readFilter(i.ctx, i.stats) - if i.err != nil { - i.data.invalidate() - return nil, base.LazyValue{} - } - mayContain := i.reader.tableFilter.mayContain(dataH.Get(), prefix) - dataH.Release() - if !mayContain { - // This invalidation may not be necessary for correctness, and may - // be a place to optimize later by reusing the already loaded - // block. It was necessary in earlier versions of the code since - // the caller was allowed to call Next when SeekPrefixGE returned - // nil. This is no longer allowed. - i.data.invalidate() - return nil, base.LazyValue{} - } - i.lastBloomFilterMatched = true - } - - // Bloom filter matches. - - // SeekPrefixGE performs various step-instead-of-seeking optimizations: eg - // enabled by trySeekUsingNext, or by monotonically increasing bounds - // (i.boundsCmp). Care must be taken to ensure that when performing these - // optimizations and the iterator becomes exhausted, - // i.maybeFilteredKeysTwoLevel is set appropriately. Consider a previous - // SeekPrefixGE that filtered keys from k until the current iterator - // position. - // - // If the previous SeekPrefixGE exhausted the iterator while seeking within - // the two-level index, it's possible keys greater than or equal to the - // current search key were filtered through skipped index blocks. We must - // not reuse the position of the two-level index iterator without - // remembering the previous value of maybeFilteredKeysTwoLevel. - - // We fall into the slow path if i.index.isDataInvalidated() even if the - // top-level iterator is already positioned correctly and all other - // conditions are met. An alternative structure could reuse topLevelIndex's - // current position and reload the index block to which it points. Arguably, - // an index block load is expensive and the index block may still be earlier - // than the index block containing the sought key, resulting in a wasteful - // block load. - - var dontSeekWithinSingleLevelIter bool - if i.topLevelIndex.isDataInvalidated() || !i.topLevelIndex.valid() || i.index.isDataInvalidated() || err != nil || - (i.boundsCmp <= 0 && !flags.TrySeekUsingNext()) || i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 { - // Slow-path: need to position the topLevelIndex. - - // The previous exhausted state of singleLevelIterator is no longer - // relevant, since we may be moving to a different index block. - i.exhaustedBounds = 0 - i.maybeFilteredKeysTwoLevel = false - flags = flags.DisableTrySeekUsingNext() - var ikey *InternalKey - if ikey, _ = i.topLevelIndex.SeekGE(key, flags); ikey == nil { - i.data.invalidate() - i.index.invalidate() - return nil, base.LazyValue{} - } - - result := i.loadIndex(+1) - if result == loadBlockFailed { - i.boundsCmp = 0 - return nil, base.LazyValue{} - } - if result == loadBlockIrrelevant { - // Enforce the upper bound here since don't want to bother moving - // to the next entry in the top level index if upper bound is - // already exceeded. Note that the next entry starts with keys >= - // ikey.UserKey since even though this is the block separator, the - // same user key can span multiple index blocks. Since upper is - // exclusive we use >= below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 - } - // Fall through to skipForward. - dontSeekWithinSingleLevelIter = true - // Clear boundsCmp. - // - // In the typical cases where dontSeekWithinSingleLevelIter=false, - // the singleLevelIterator.SeekPrefixGE call will clear boundsCmp. - // However, in this case where dontSeekWithinSingleLevelIter=true, - // we never seek on the single-level iterator. This call will fall - // through to skipForward, which may improperly leave boundsCmp=+1 - // unless we clear it here. - i.boundsCmp = 0 - } - } else { - // INVARIANT: err == nil. - // - // Else fast-path: There are two possible cases, from - // (i.boundsCmp > 0 || flags.TrySeekUsingNext()): - // - // 1) The bounds have moved forward (i.boundsCmp > 0) and this - // SeekPrefixGE is respecting the lower bound (guaranteed by Iterator). We - // know that the iterator must already be positioned within or just - // outside the previous bounds. Therefore, the topLevelIndex iter cannot - // be positioned at an entry ahead of the seek position (though it can be - // positioned behind). The !i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 - // confirms that it is not behind. Since it is not ahead and not behind it - // must be at the right position. - // - // 2) This SeekPrefixGE will land on a key that is greater than the key we - // are currently at (guaranteed by trySeekUsingNext), but since i.cmp(key, - // i.topLevelIndex.Key().UserKey) <= 0, we are at the correct lower level - // index block. No need to reset the state of singleLevelIterator. - // - // Note that cases 1 and 2 never overlap, and one of them must be true. - // This invariant checking is important enough that we do not gate it - // behind invariants.Enabled. - if i.boundsCmp > 0 == flags.TrySeekUsingNext() { - panic(fmt.Sprintf("inconsistency in optimization case 1 %t and case 2 %t", - i.boundsCmp > 0, flags.TrySeekUsingNext())) - } - - if !flags.TrySeekUsingNext() { - // Case 1. Bounds have changed so the previous exhausted bounds state is - // irrelevant. - // WARNING-data-exhausted: this is safe to do only because the monotonic - // bounds optimizations only work when !data-exhausted. If they also - // worked with data-exhausted, we have made it unclear whether - // data-exhausted is actually true. See the comment at the top of the - // file. - i.exhaustedBounds = 0 - } - // Else flags.TrySeekUsingNext(). The i.exhaustedBounds is important to - // preserve for singleLevelIterator, and twoLevelIterator.skipForward. See - // bug https://github.com/cockroachdb/pebble/issues/2036. - } - - if !dontSeekWithinSingleLevelIter { - if ikey, val := i.singleLevelIterator.seekPrefixGE( - prefix, key, flags, false /* checkFilter */); ikey != nil { - return ikey, val - } - } - // NB: skipForward checks whether exhaustedBounds is already +1. - return i.skipForward() -} - -// SeekLT implements internalIterator.SeekLT, as documented in the pebble -// package. Note that SeekLT only checks the lower bound. It is up to the -// caller to ensure that key is less than the upper bound. -func (i *twoLevelIterator) SeekLT( - key []byte, flags base.SeekLTFlags, -) (*InternalKey, base.LazyValue) { - i.exhaustedBounds = 0 - i.err = nil // clear cached iteration error - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - - var result loadBlockResult - var ikey *InternalKey - // NB: Unlike SeekGE, we don't have a fast-path here since we don't know - // whether the topLevelIndex is positioned after the position that would - // be returned by doing i.topLevelIndex.SeekGE(). To know this we would - // need to know the index key preceding the current one. - // NB: If a bound-limited block property filter is configured, it's - // externally ensured that the filter is disabled (through returning - // Intersects=false irrespective of the block props provided) during seeks. - i.maybeFilteredKeysTwoLevel = false - if ikey, _ = i.topLevelIndex.SeekGE(key, base.SeekGEFlagsNone); ikey == nil { - if ikey, _ = i.topLevelIndex.Last(); ikey == nil { - i.data.invalidate() - i.index.invalidate() - return nil, base.LazyValue{} - } - - result = i.loadIndex(-1) - if result == loadBlockFailed { - return nil, base.LazyValue{} - } - if result == loadBlockOK { - if ikey, val := i.singleLevelIterator.lastInternal(); ikey != nil { - return ikey, val - } - // Fall through to skipBackward since the singleLevelIterator did - // not have any blocks that satisfy the block interval - // constraints, or the lower bound was reached. - } - // Else loadBlockIrrelevant, so fall through. - } else { - result = i.loadIndex(-1) - if result == loadBlockFailed { - return nil, base.LazyValue{} - } - if result == loadBlockOK { - if ikey, val := i.singleLevelIterator.SeekLT(key, flags); ikey != nil { - return ikey, val - } - // Fall through to skipBackward since the singleLevelIterator did - // not have any blocks that satisfy the block interval - // constraint, or the lower bound was reached. - } - // Else loadBlockIrrelevant, so fall through. - } - if result == loadBlockIrrelevant { - // Enforce the lower bound here since don't want to bother moving to - // the previous entry in the top level index if lower bound is already - // exceeded. Note that the previous entry starts with keys <= - // ikey.UserKey since even though this is the current block's - // separator, the same user key can span multiple index blocks. - if i.lower != nil && i.cmp(ikey.UserKey, i.lower) < 0 { - i.exhaustedBounds = -1 - } - } - // NB: skipBackward checks whether exhaustedBounds is already -1. - return i.skipBackward() -} - -// First implements internalIterator.First, as documented in the pebble -// package. Note that First only checks the upper bound. It is up to the caller -// to ensure that key is greater than or equal to the lower bound (e.g. via a -// call to SeekGE(lower)). -func (i *twoLevelIterator) First() (*InternalKey, base.LazyValue) { - if i.lower != nil { - panic("twoLevelIterator.First() used despite lower bound") - } - i.exhaustedBounds = 0 - i.maybeFilteredKeysTwoLevel = false - i.err = nil // clear cached iteration error - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - - var ikey *InternalKey - if ikey, _ = i.topLevelIndex.First(); ikey == nil { - return nil, base.LazyValue{} - } - - result := i.loadIndex(+1) - if result == loadBlockFailed { - return nil, base.LazyValue{} - } - if result == loadBlockOK { - if ikey, val := i.singleLevelIterator.First(); ikey != nil { - return ikey, val - } - // Else fall through to skipForward. - } else { - // result == loadBlockIrrelevant. Enforce the upper bound here since - // don't want to bother moving to the next entry in the top level - // index if upper bound is already exceeded. Note that the next entry - // starts with keys >= ikey.UserKey since even though this is the - // block separator, the same user key can span multiple index blocks. - // Since upper is exclusive we use >= below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 - } - } - // NB: skipForward checks whether exhaustedBounds is already +1. - return i.skipForward() -} - -// Last implements internalIterator.Last, as documented in the pebble -// package. Note that Last only checks the lower bound. It is up to the caller -// to ensure that key is less than the upper bound (e.g. via a call to -// SeekLT(upper)) -func (i *twoLevelIterator) Last() (*InternalKey, base.LazyValue) { - if i.upper != nil { - panic("twoLevelIterator.Last() used despite upper bound") - } - i.exhaustedBounds = 0 - i.maybeFilteredKeysTwoLevel = false - i.err = nil // clear cached iteration error - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - - var ikey *InternalKey - if ikey, _ = i.topLevelIndex.Last(); ikey == nil { - return nil, base.LazyValue{} - } - - result := i.loadIndex(-1) - if result == loadBlockFailed { - return nil, base.LazyValue{} - } - if result == loadBlockOK { - if ikey, val := i.singleLevelIterator.Last(); ikey != nil { - return ikey, val - } - // Else fall through to skipBackward. - } else { - // result == loadBlockIrrelevant. Enforce the lower bound here - // since don't want to bother moving to the previous entry in the - // top level index if lower bound is already exceeded. Note that - // the previous entry starts with keys <= ikey.UserKey since even - // though this is the current block's separator, the same user key - // can span multiple index blocks. - if i.lower != nil && i.cmp(ikey.UserKey, i.lower) < 0 { - i.exhaustedBounds = -1 - } - } - // NB: skipBackward checks whether exhaustedBounds is already -1. - return i.skipBackward() -} - -// Next implements internalIterator.Next, as documented in the pebble -// package. -// Note: twoLevelCompactionIterator.Next mirrors the implementation of -// twoLevelIterator.Next due to performance. Keep the two in sync. -func (i *twoLevelIterator) Next() (*InternalKey, base.LazyValue) { - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - i.maybeFilteredKeysTwoLevel = false - if i.err != nil { - return nil, base.LazyValue{} - } - if key, val := i.singleLevelIterator.Next(); key != nil { - return key, val - } - return i.skipForward() -} - -// NextPrefix implements (base.InternalIterator).NextPrefix. -func (i *twoLevelIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { - if i.exhaustedBounds == +1 { - panic("Next called even though exhausted upper bound") - } - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - i.maybeFilteredKeysTwoLevel = false - if i.err != nil { - return nil, base.LazyValue{} - } - if key, val := i.singleLevelIterator.NextPrefix(succKey); key != nil { - return key, val - } - // Did not find prefix in the existing second-level index block. This is the - // slow-path where we seek the iterator. - var ikey *InternalKey - if ikey, _ = i.topLevelIndex.SeekGE(succKey, base.SeekGEFlagsNone); ikey == nil { - i.data.invalidate() - i.index.invalidate() - return nil, base.LazyValue{} - } - result := i.loadIndex(+1) - if result == loadBlockFailed { - return nil, base.LazyValue{} - } - if result == loadBlockIrrelevant { - // Enforce the upper bound here since don't want to bother moving to the - // next entry in the top level index if upper bound is already exceeded. - // Note that the next entry starts with keys >= ikey.UserKey since even - // though this is the block separator, the same user key can span multiple - // index blocks. Since upper is exclusive we use >= below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 - } - } else if key, val := i.singleLevelIterator.SeekGE(succKey, base.SeekGEFlagsNone); key != nil { - return key, val - } - return i.skipForward() -} - -// Prev implements internalIterator.Prev, as documented in the pebble -// package. -func (i *twoLevelIterator) Prev() (*InternalKey, base.LazyValue) { - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - i.maybeFilteredKeysTwoLevel = false - if i.err != nil { - return nil, base.LazyValue{} - } - if key, val := i.singleLevelIterator.Prev(); key != nil { - return key, val - } - return i.skipBackward() -} - -func (i *twoLevelIterator) skipForward() (*InternalKey, base.LazyValue) { - for { - if i.err != nil || i.exhaustedBounds > 0 { - return nil, base.LazyValue{} - } - i.exhaustedBounds = 0 - var ikey *InternalKey - if ikey, _ = i.topLevelIndex.Next(); ikey == nil { - i.data.invalidate() - i.index.invalidate() - return nil, base.LazyValue{} - } - result := i.loadIndex(+1) - if result == loadBlockFailed { - return nil, base.LazyValue{} - } - if result == loadBlockOK { - if ikey, val := i.singleLevelIterator.firstInternal(); ikey != nil { - return ikey, val - } - // Next iteration will return if singleLevelIterator set - // exhaustedBounds = +1. - } else { - // result == loadBlockIrrelevant. Enforce the upper bound here - // since don't want to bother moving to the next entry in the top - // level index if upper bound is already exceeded. Note that the - // next entry starts with keys >= ikey.UserKey since even though - // this is the block separator, the same user key can span - // multiple index blocks. Since upper is exclusive we use >= - // below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 - // Next iteration will return. - } - } - } -} - -func (i *twoLevelIterator) skipBackward() (*InternalKey, base.LazyValue) { - for { - if i.err != nil || i.exhaustedBounds < 0 { - return nil, base.LazyValue{} - } - i.exhaustedBounds = 0 - var ikey *InternalKey - if ikey, _ = i.topLevelIndex.Prev(); ikey == nil { - i.data.invalidate() - i.index.invalidate() - return nil, base.LazyValue{} - } - result := i.loadIndex(-1) - if result == loadBlockFailed { - return nil, base.LazyValue{} - } - if result == loadBlockOK { - if ikey, val := i.singleLevelIterator.lastInternal(); ikey != nil { - return ikey, val - } - // Next iteration will return if singleLevelIterator set - // exhaustedBounds = -1. - } else { - // result == loadBlockIrrelevant. Enforce the lower bound here - // since don't want to bother moving to the previous entry in the - // top level index if lower bound is already exceeded. Note that - // the previous entry starts with keys <= ikey.UserKey since even - // though this is the current block's separator, the same user key - // can span multiple index blocks. - if i.lower != nil && i.cmp(ikey.UserKey, i.lower) < 0 { - i.exhaustedBounds = -1 - // Next iteration will return. - } - } - } -} - -// Close implements internalIterator.Close, as documented in the pebble -// package. -func (i *twoLevelIterator) Close() error { - var err error - if i.closeHook != nil { - err = firstError(err, i.closeHook(i)) - } - err = firstError(err, i.data.Close()) - err = firstError(err, i.index.Close()) - err = firstError(err, i.topLevelIndex.Close()) - if i.dataRH != nil { - err = firstError(err, i.dataRH.Close()) - i.dataRH = nil - } - err = firstError(err, i.err) - if i.bpfs != nil { - releaseBlockPropertiesFilterer(i.bpfs) - } - if i.vbReader != nil { - i.vbReader.close() - } - if i.vbRH != nil { - err = firstError(err, i.vbRH.Close()) - i.vbRH = nil - } - *i = twoLevelIterator{ - singleLevelIterator: i.singleLevelIterator.resetForReuse(), - topLevelIndex: i.topLevelIndex.resetForReuse(), - } - twoLevelIterPool.Put(i) - return err -} - -// Note: twoLevelCompactionIterator and compactionIterator are very similar but -// were separated due to performance. -type twoLevelCompactionIterator struct { - *twoLevelIterator - bytesIterated *uint64 - prevOffset uint64 -} - -// twoLevelCompactionIterator implements the base.InternalIterator interface. -var _ base.InternalIterator = (*twoLevelCompactionIterator)(nil) - -func (i *twoLevelCompactionIterator) Close() error { - return i.twoLevelIterator.Close() -} - -func (i *twoLevelCompactionIterator) SeekGE( - key []byte, flags base.SeekGEFlags, -) (*InternalKey, base.LazyValue) { - panic("pebble: SeekGE unimplemented") -} - -func (i *twoLevelCompactionIterator) SeekPrefixGE( - prefix, key []byte, flags base.SeekGEFlags, -) (*base.InternalKey, base.LazyValue) { - panic("pebble: SeekPrefixGE unimplemented") -} - -func (i *twoLevelCompactionIterator) SeekLT( - key []byte, flags base.SeekLTFlags, -) (*InternalKey, base.LazyValue) { - panic("pebble: SeekLT unimplemented") -} - -func (i *twoLevelCompactionIterator) First() (*InternalKey, base.LazyValue) { - i.err = nil // clear cached iteration error - return i.skipForward(i.twoLevelIterator.First()) -} - -func (i *twoLevelCompactionIterator) Last() (*InternalKey, base.LazyValue) { - panic("pebble: Last unimplemented") -} - -// Note: twoLevelCompactionIterator.Next mirrors the implementation of -// twoLevelIterator.Next due to performance. Keep the two in sync. -func (i *twoLevelCompactionIterator) Next() (*InternalKey, base.LazyValue) { - if i.err != nil { - return nil, base.LazyValue{} - } - return i.skipForward(i.singleLevelIterator.Next()) -} - -func (i *twoLevelCompactionIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { - panic("pebble: NextPrefix unimplemented") -} - -func (i *twoLevelCompactionIterator) Prev() (*InternalKey, base.LazyValue) { - panic("pebble: Prev unimplemented") -} - -func (i *twoLevelCompactionIterator) String() string { - return i.reader.fileNum.String() -} - -func (i *twoLevelCompactionIterator) skipForward( - key *InternalKey, val base.LazyValue, -) (*InternalKey, base.LazyValue) { - if key == nil { - for { - if key, _ := i.topLevelIndex.Next(); key == nil { - break - } - result := i.loadIndex(+1) - if result != loadBlockOK { - if i.err != nil { - break - } - switch result { - case loadBlockFailed: - // We checked that i.index was at a valid entry, so - // loadBlockFailed could not have happened due to to i.index - // being exhausted, and must be due to an error. - panic("loadBlock should not have failed with no error") - case loadBlockIrrelevant: - panic("compactionIter should not be using block intervals for skipping") - default: - panic(fmt.Sprintf("unexpected case %d", result)) - } - } - // result == loadBlockOK - if key, val = i.singleLevelIterator.First(); key != nil { - break - } - } - } - - curOffset := i.recordOffset() - *i.bytesIterated += uint64(curOffset - i.prevOffset) - i.prevOffset = curOffset - return key, val -} - type blockTransform func([]byte) ([]byte, error) // ReaderOption provide an interface to do work on Reader while it is being @@ -2551,7 +129,7 @@ func (m Mergers) readerApply(r *Reader) { // number. If not specified, a unique cache ID will be used. type cacheOpts struct { cacheID uint64 - fileNum base.FileNum + fileNum base.DiskFileNum } // Marker function to indicate the option should be applied before reading the @@ -2563,7 +141,7 @@ func (c *cacheOpts) readerApply(r *Reader) { if r.cacheID == 0 { r.cacheID = c.cacheID } - if r.fileNum == 0 { + if r.fileNum.FileNum() == 0 { r.fileNum = c.fileNum } } @@ -2572,7 +150,7 @@ func (c *cacheOpts) writerApply(w *Writer) { if w.cacheID == 0 { w.cacheID = c.cacheID } - if w.fileNum == 0 { + if w.fileNum.FileNum() == 0 { w.fileNum = c.fileNum } } @@ -2590,17 +168,39 @@ func (rawTombstonesOpt) readerApply(r *Reader) { } func init() { - private.SSTableCacheOpts = func(cacheID uint64, fileNum base.FileNum) interface{} { + private.SSTableCacheOpts = func(cacheID uint64, fileNum base.DiskFileNum) interface{} { return &cacheOpts{cacheID, fileNum} } private.SSTableRawTombstonesOpt = rawTombstonesOpt{} } +// CommonReader abstracts functionality over a Reader or a VirtualReader. This +// can be used by code which doesn't care to distinguish between a reader and a +// virtual reader. +type CommonReader interface { + NewRawRangeKeyIter() (keyspan.FragmentIterator, error) + NewRawRangeDelIter() (keyspan.FragmentIterator, error) + NewIterWithBlockPropertyFiltersAndContextEtc( + ctx context.Context, lower, upper []byte, + filterer *BlockPropertiesFilterer, + hideObsoletePoints, useFilterBlock bool, + stats *base.InternalIteratorStats, + rp ReaderProvider, + ) (Iterator, error) + NewCompactionIter( + bytesIterated *uint64, + rp ReaderProvider, + bufferPool *BufferPool, + ) (Iterator, error) + EstimateDiskUsage(start, end []byte) (uint64, error) + CommonProperties() *CommonProperties +} + // Reader is a table reader. type Reader struct { readable objstorage.Readable cacheID uint64 - fileNum base.FileNum + fileNum base.DiskFileNum err error indexBH BlockHandle filterBH BlockHandle @@ -2623,6 +223,14 @@ type Reader struct { rawTombstones bool mergerOK bool checksumType ChecksumType + // metaBufferPool is a buffer pool used exclusively when opening a table and + // loading its meta blocks. metaBufferPoolAlloc is used to batch-allocate + // the BufferPool.pool slice as a part of the Reader allocation. It's + // capacity 3 to accommodate the meta block (1), and both the compressed + // properties block (1) and decompressed properties block (1) + // simultaneously. + metaBufferPool BufferPool + metaBufferPoolAlloc [3]allocedBuffer } // Close implements DB.Close, as documented in the pebble package. @@ -2652,28 +260,64 @@ func (r *Reader) NewIterWithBlockPropertyFilters( stats *base.InternalIteratorStats, rp ReaderProvider, ) (Iterator, error) { - return r.NewIterWithBlockPropertyFiltersAndContext(context.Background(), lower, upper, filterer, - useFilterBlock, stats, rp) + return r.newIterWithBlockPropertyFiltersAndContext( + context.Background(), + lower, upper, filterer, false, useFilterBlock, stats, rp, nil, + ) } -// NewIterWithBlockPropertyFiltersAndContext is similar to +// NewIterWithBlockPropertyFiltersAndContextEtc is similar to // NewIterWithBlockPropertyFilters and additionally accepts a context for // tracing. -func (r *Reader) NewIterWithBlockPropertyFiltersAndContext( +// +// If hideObsoletePoints, the callee assumes that filterer already includes +// obsoleteKeyBlockPropertyFilter. The caller can satisfy this contract by +// first calling TryAddBlockPropertyFilterForHideObsoletePoints. +func (r *Reader) NewIterWithBlockPropertyFiltersAndContextEtc( ctx context.Context, lower, upper []byte, filterer *BlockPropertiesFilterer, - useFilterBlock bool, + hideObsoletePoints, useFilterBlock bool, stats *base.InternalIteratorStats, rp ReaderProvider, ) (Iterator, error) { + return r.newIterWithBlockPropertyFiltersAndContext( + ctx, lower, upper, filterer, hideObsoletePoints, useFilterBlock, stats, rp, nil, + ) +} +// TryAddBlockPropertyFilterForHideObsoletePoints is expected to be called +// before the call to NewIterWithBlockPropertyFiltersAndContextEtc, to get the +// value of hideObsoletePoints and potentially add a block property filter. +func (r *Reader) TryAddBlockPropertyFilterForHideObsoletePoints( + snapshotForHideObsoletePoints uint64, + fileLargestSeqNum uint64, + pointKeyFilters []BlockPropertyFilter, +) (hideObsoletePoints bool, filters []BlockPropertyFilter) { + hideObsoletePoints = r.tableFormat >= TableFormatPebblev4 && + snapshotForHideObsoletePoints > fileLargestSeqNum + if hideObsoletePoints { + pointKeyFilters = append(pointKeyFilters, obsoleteKeyBlockPropertyFilter{}) + } + return hideObsoletePoints, pointKeyFilters +} + +func (r *Reader) newIterWithBlockPropertyFiltersAndContext( + ctx context.Context, + lower, upper []byte, + filterer *BlockPropertiesFilterer, + hideObsoletePoints bool, + useFilterBlock bool, + stats *base.InternalIteratorStats, + rp ReaderProvider, + v *virtualState, +) (Iterator, error) { // NB: pebble.tableCache wraps the returned iterator with one which performs // reference counting on the Reader, preventing the Reader from being closed // until the final iterator closes. if r.Properties.IndexType == twoLevelIndex { i := twoLevelIterPool.Get().(*twoLevelIterator) - err := i.init(ctx, r, lower, upper, filterer, useFilterBlock, stats, rp) + err := i.init(ctx, r, v, lower, upper, filterer, useFilterBlock, hideObsoletePoints, stats, rp, nil /* bufferPool */) if err != nil { return nil, err } @@ -2681,7 +325,7 @@ func (r *Reader) NewIterWithBlockPropertyFiltersAndContext( } i := singleLevelIterPool.Get().(*singleLevelIterator) - err := i.init(ctx, r, lower, upper, filterer, useFilterBlock, stats, rp) + err := i.init(ctx, r, v, lower, upper, filterer, useFilterBlock, hideObsoletePoints, stats, rp, nil /* bufferPool */) if err != nil { return nil, err } @@ -2701,11 +345,23 @@ func (r *Reader) NewIter(lower, upper []byte) (Iterator, error) { // NewCompactionIter returns an iterator similar to NewIter but it also increments // the number of bytes iterated. If an error occurs, NewCompactionIter cleans up // after itself and returns a nil iterator. -func (r *Reader) NewCompactionIter(bytesIterated *uint64, rp ReaderProvider) (Iterator, error) { +func (r *Reader) NewCompactionIter( + bytesIterated *uint64, rp ReaderProvider, bufferPool *BufferPool, +) (Iterator, error) { + return r.newCompactionIter(bytesIterated, rp, nil, bufferPool) +} + +func (r *Reader) newCompactionIter( + bytesIterated *uint64, rp ReaderProvider, v *virtualState, bufferPool *BufferPool, +) (Iterator, error) { if r.Properties.IndexType == twoLevelIndex { i := twoLevelIterPool.Get().(*twoLevelIterator) - err := i.init(context.Background(), r, nil /* lower */, nil, /* upper */ - nil, false /* useFilter */, nil /* stats */, rp) + err := i.init( + context.Background(), + r, v, nil /* lower */, nil /* upper */, nil, + false /* useFilter */, v != nil && v.isForeign, /* hideObsoletePoints */ + nil /* stats */, rp, bufferPool, + ) if err != nil { return nil, err } @@ -2716,8 +372,11 @@ func (r *Reader) NewCompactionIter(bytesIterated *uint64, rp ReaderProvider) (It }, nil } i := singleLevelIterPool.Get().(*singleLevelIterator) - err := i.init(context.Background(), r, nil /* lower */, nil, /* upper */ - nil, false /* useFilter */, nil /* stats */, rp) + err := i.init( + context.Background(), r, v, nil /* lower */, nil, /* upper */ + nil, false /* useFilter */, v != nil && v.isForeign, /* hideObsoletePoints */ + nil /* stats */, rp, bufferPool, + ) if err != nil { return nil, err } @@ -2742,8 +401,8 @@ func (r *Reader) NewRawRangeDelIter() (keyspan.FragmentIterator, error) { if err != nil { return nil, err } - i := &fragmentBlockIter{} - if err := i.blockIter.initHandle(r.Compare, h, r.Properties.GlobalSeqNum); err != nil { + i := &fragmentBlockIter{elideSameSeqnum: true} + if err := i.blockIter.initHandle(r.Compare, h, r.Properties.GlobalSeqNum, false); err != nil { return nil, err } return i, nil @@ -2764,7 +423,7 @@ func (r *Reader) NewRawRangeKeyIter() (keyspan.FragmentIterator, error) { return nil, err } i := rangeKeyFragmentBlockIterPool.Get().(*rangeKeyFragmentBlockIter) - if err := i.blockIter.initHandle(r.Compare, h, r.Properties.GlobalSeqNum); err != nil { + if err := i.blockIter.initHandle(r.Compare, h, r.Properties.GlobalSeqNum, false); err != nil { return nil, err } return i, nil @@ -2783,24 +442,26 @@ func (i *rangeKeyFragmentBlockIter) Close() error { func (r *Reader) readIndex( ctx context.Context, stats *base.InternalIteratorStats, -) (cache.Handle, error) { - return r.readBlock(ctx, r.indexBH, nil, nil, stats) +) (bufferHandle, error) { + ctx = objiotracing.WithBlockType(ctx, objiotracing.MetadataBlock) + return r.readBlock(ctx, r.indexBH, nil, nil, stats, nil /* buffer pool */) } func (r *Reader) readFilter( ctx context.Context, stats *base.InternalIteratorStats, -) (cache.Handle, error) { - return r.readBlock(ctx, r.filterBH, nil /* transform */, nil /* readHandle */, stats) +) (bufferHandle, error) { + ctx = objiotracing.WithBlockType(ctx, objiotracing.FilterBlock) + return r.readBlock(ctx, r.filterBH, nil /* transform */, nil /* readHandle */, stats, nil /* buffer pool */) } -func (r *Reader) readRangeDel(stats *base.InternalIteratorStats) (cache.Handle, error) { - return r.readBlock( - context.Background(), r.rangeDelBH, r.rangeDelTransform, nil /* readHandle */, stats) +func (r *Reader) readRangeDel(stats *base.InternalIteratorStats) (bufferHandle, error) { + ctx := objiotracing.WithBlockType(context.Background(), objiotracing.MetadataBlock) + return r.readBlock(ctx, r.rangeDelBH, r.rangeDelTransform, nil /* readHandle */, stats, nil /* buffer pool */) } -func (r *Reader) readRangeKey(stats *base.InternalIteratorStats) (cache.Handle, error) { - return r.readBlock( - context.Background(), r.rangeKeyBH, nil /* transform */, nil /* readHandle */, stats) +func (r *Reader) readRangeKey(stats *base.InternalIteratorStats) (bufferHandle, error) { + ctx := objiotracing.WithBlockType(context.Background(), objiotracing.MetadataBlock) + return r.readBlock(ctx, r.rangeKeyBH, nil /* transform */, nil /* readHandle */, stats, nil /* buffer pool */) } func checkChecksum( @@ -2825,33 +486,76 @@ func checkChecksum( return nil } -// readBlock reads and decompresses a block from disk into memory. +type cacheValueOrBuf struct { + // buf.Valid() returns true if backed by a BufferPool. + buf Buf + // v is non-nil if backed by the block cache. + v *cache.Value +} + +func (b cacheValueOrBuf) get() []byte { + if b.buf.Valid() { + return b.buf.p.pool[b.buf.i].b + } + return b.v.Buf() +} + +func (b cacheValueOrBuf) release() { + if b.buf.Valid() { + b.buf.Release() + } else { + cache.Free(b.v) + } +} + +func (b cacheValueOrBuf) truncate(n int) { + if b.buf.Valid() { + b.buf.p.pool[b.buf.i].b = b.buf.p.pool[b.buf.i].b[:n] + } else { + b.v.Truncate(n) + } +} + func (r *Reader) readBlock( ctx context.Context, bh BlockHandle, transform blockTransform, readHandle objstorage.ReadHandle, stats *base.InternalIteratorStats, -) (handle cache.Handle, _ error) { + bufferPool *BufferPool, +) (handle bufferHandle, _ error) { if h := r.opts.Cache.Get(r.cacheID, r.fileNum, bh.Offset); h.Get() != nil { + // Cache hit. if readHandle != nil { - readHandle.RecordCacheHit(int64(bh.Offset), int64(bh.Length+blockTrailerLen)) + readHandle.RecordCacheHit(ctx, int64(bh.Offset), int64(bh.Length+blockTrailerLen)) } if stats != nil { stats.BlockBytes += bh.Length stats.BlockBytesInCache += bh.Length } - return h, nil + // This block is already in the cache; return a handle to existing vlaue + // in the cache. + return bufferHandle{h: h}, nil + } + + // Cache miss. + var compressed cacheValueOrBuf + if bufferPool != nil { + compressed = cacheValueOrBuf{ + buf: bufferPool.Alloc(int(bh.Length + blockTrailerLen)), + } + } else { + compressed = cacheValueOrBuf{ + v: cache.Alloc(int(bh.Length + blockTrailerLen)), + } } - v := r.opts.Cache.Alloc(int(bh.Length + blockTrailerLen)) - b := v.Buf() readStartTime := time.Now() var err error if readHandle != nil { - _, err = readHandle.ReadAt(b, int64(bh.Offset)) + err = readHandle.ReadAt(ctx, compressed.get(), int64(bh.Offset)) } else { - _, err = r.readable.ReadAt(b, int64(bh.Offset)) + err = r.readable.ReadAt(ctx, compressed.get(), int64(bh.Offset)) } readDuration := time.Since(readStartTime) // TODO(sumeer): should the threshold be configurable. @@ -2864,56 +568,74 @@ func (r *Reader) readBlock( // interface{}, unless necessary. if readDuration >= slowReadTracingThreshold && r.opts.LoggerAndTracer.IsTracingEnabled(ctx) { r.opts.LoggerAndTracer.Eventf(ctx, "reading %d bytes took %s", - bh.Length+blockTrailerLen, readDuration.String()) + int(bh.Length+blockTrailerLen), readDuration.String()) } if stats != nil { stats.BlockReadDuration += readDuration } if err != nil { - r.opts.Cache.Free(v) - return cache.Handle{}, err + compressed.release() + return bufferHandle{}, err + } + if err := checkChecksum(r.checksumType, compressed.get(), bh, r.fileNum.FileNum()); err != nil { + compressed.release() + return bufferHandle{}, err } - if err := checkChecksum(r.checksumType, b, bh, r.fileNum); err != nil { - r.opts.Cache.Free(v) - return cache.Handle{}, err - } + typ := blockType(compressed.get()[bh.Length]) + compressed.truncate(int(bh.Length)) - typ := blockType(b[bh.Length]) - b = b[:bh.Length] - v.Truncate(len(b)) + var decompressed cacheValueOrBuf + if typ == noCompressionBlockType { + decompressed = compressed + } else { + // Decode the length of the decompressed value. + decodedLen, prefixLen, err := decompressedLen(typ, compressed.get()) + if err != nil { + compressed.release() + return bufferHandle{}, err + } - decoded, err := decompressBlock(r.opts.Cache, typ, b) - if decoded != nil { - r.opts.Cache.Free(v) - v = decoded - b = v.Buf() - } else if err != nil { - r.opts.Cache.Free(v) - return cache.Handle{}, err + if bufferPool != nil { + decompressed = cacheValueOrBuf{buf: bufferPool.Alloc(decodedLen)} + } else { + decompressed = cacheValueOrBuf{v: cache.Alloc(decodedLen)} + } + if _, err := decompressInto(typ, compressed.get()[prefixLen:], decompressed.get()); err != nil { + compressed.release() + return bufferHandle{}, err + } + compressed.release() } if transform != nil { - // Transforming blocks is rare, so the extra copy of the transformed data - // is not problematic. - var err error - b, err = transform(b) + // Transforming blocks is very rare, so the extra copy of the + // transformed data is not problematic. + tmpTransformed, err := transform(decompressed.get()) if err != nil { - r.opts.Cache.Free(v) - return cache.Handle{}, err + decompressed.release() + return bufferHandle{}, err } - newV := r.opts.Cache.Alloc(len(b)) - copy(newV.Buf(), b) - r.opts.Cache.Free(v) - v = newV + + var transformed cacheValueOrBuf + if bufferPool != nil { + transformed = cacheValueOrBuf{buf: bufferPool.Alloc(len(tmpTransformed))} + } else { + transformed = cacheValueOrBuf{v: cache.Alloc(len(tmpTransformed))} + } + copy(transformed.get(), tmpTransformed) + decompressed.release() + decompressed = transformed } if stats != nil { stats.BlockBytes += bh.Length } - - h := r.opts.Cache.Set(r.cacheID, r.fileNum, bh.Offset, v) - return h, nil + if decompressed.buf.Valid() { + return bufferHandle{b: decompressed.buf}, nil + } + h := r.opts.Cache.Set(r.cacheID, r.fileNum, bh.Offset, decompressed.v) + return bufferHandle{h: h}, nil } func (r *Reader) transformRangeDelV1(b []byte) ([]byte, error) { @@ -2922,7 +644,7 @@ func (r *Reader) transformRangeDelV1(b []byte) ([]byte, error) { // tombstones. We need properly fragmented and sorted range tombstones in // order to serve from them directly. iter := &blockIter{} - if err := iter.init(r.Compare, b, r.Properties.GlobalSeqNum); err != nil { + if err := iter.init(r.Compare, b, r.Properties.GlobalSeqNum, false); err != nil { return nil, err } var tombstones []keyspan.Span @@ -2960,8 +682,21 @@ func (r *Reader) transformRangeDelV1(b []byte) ([]byte, error) { } func (r *Reader) readMetaindex(metaindexBH BlockHandle) error { + // We use a BufferPool when reading metaindex blocks in order to avoid + // populating the block cache with these blocks. In heavy-write workloads, + // especially with high compaction concurrency, new tables may be created + // frequently. Populating the block cache with these metaindex blocks adds + // additional contention on the block cache mutexes (see #1997). + // Additionally, these blocks are exceedingly unlikely to be read again + // while they're still in the block cache except in misconfigurations with + // excessive sstables counts or a table cache that's far too small. + r.metaBufferPool.initPreallocated(r.metaBufferPoolAlloc[:0]) + // When we're finished, release the buffers we've allocated back to memory + // allocator. We don't expect to use metaBufferPool again. + defer r.metaBufferPool.Release() + b, err := r.readBlock( - context.Background(), metaindexBH, nil /* transform */, nil /* readHandle */, nil /* stats */) + context.Background(), metaindexBH, nil /* transform */, nil /* readHandle */, nil /* stats */, &r.metaBufferPool) if err != nil { return err } @@ -3004,12 +739,12 @@ func (r *Reader) readMetaindex(metaindexBH BlockHandle) error { if bh, ok := meta[metaPropertiesName]; ok { b, err = r.readBlock( - context.Background(), bh, nil /* transform */, nil /* readHandle */, nil /* stats */) + context.Background(), bh, nil /* transform */, nil /* readHandle */, nil /* stats */, nil /* buffer pool */) if err != nil { return err } r.propertiesBH = bh - err := r.Properties.load(b.Get(), bh.Offset) + err := r.Properties.load(b.Get(), bh.Offset, r.opts.DeniedUserProperties) b.Release() if err != nil { return err @@ -3109,12 +844,13 @@ func (r *Reader) Layout() (*Layout, error) { } l.Index = append(l.Index, indexBH.BlockHandle) - subIndex, err := r.readBlock(context.Background(), - indexBH.BlockHandle, nil /* transform */, nil /* readHandle */, nil /* stats */) + subIndex, err := r.readBlock(context.Background(), indexBH.BlockHandle, + nil /* transform */, nil /* readHandle */, nil /* stats */, nil /* buffer pool */) if err != nil { return nil, err } - if err := iter.init(r.Compare, subIndex.Get(), 0 /* globalSeqNum */); err != nil { + if err := iter.init(r.Compare, subIndex.Get(), 0, /* globalSeqNum */ + false /* hideObsoletePoints */); err != nil { return nil, err } for key, value := iter.First(); key != nil; key, value = iter.Next() { @@ -3132,7 +868,7 @@ func (r *Reader) Layout() (*Layout, error) { } } if r.valueBIH.h.Length != 0 { - vbiH, err := r.readBlock(context.Background(), r.valueBIH.h, nil, nil, nil) + vbiH, err := r.readBlock(context.Background(), r.valueBIH.h, nil, nil, nil, nil /* buffer pool */) if err != nil { return nil, err } @@ -3193,7 +929,7 @@ func (r *Reader) ValidateBlockChecksums() error { // Check all blocks sequentially. Make use of read-ahead, given we are // scanning the entire file from start to end. - rh := r.readable.NewReadHandle() + rh := r.readable.NewReadHandle(context.TODO()) defer rh.Close() for _, bh := range blocks { @@ -3203,7 +939,7 @@ func (r *Reader) ValidateBlockChecksums() error { } // Read the block, which validates the checksum. - h, err := r.readBlock(context.Background(), bh, nil, rh, nil) + h, err := r.readBlock(context.Background(), bh, nil, rh, nil, nil /* buffer pool */) if err != nil { return err } @@ -3213,6 +949,11 @@ func (r *Reader) ValidateBlockChecksums() error { return nil } +// CommonProperties implemented the CommonReader interface. +func (r *Reader) CommonProperties() *CommonProperties { + return &r.Properties.CommonProperties +} + // EstimateDiskUsage returns the total size of data blocks overlapping the range // `[start, end]`. Even if a data block partially overlaps, or we cannot // determine overlap due to abbreviated index keys, the full data block size is @@ -3265,8 +1006,8 @@ func (r *Reader) EstimateDiskUsage(start, end []byte) (uint64, error) { if err != nil { return 0, errCorruptIndexEntry } - startIdxBlock, err := r.readBlock(context.Background(), - startIdxBH.BlockHandle, nil /* transform */, nil /* readHandle */, nil /* stats */) + startIdxBlock, err := r.readBlock(context.Background(), startIdxBH.BlockHandle, + nil /* transform */, nil /* readHandle */, nil /* stats */, nil /* buffer pool */) if err != nil { return 0, err } @@ -3287,7 +1028,7 @@ func (r *Reader) EstimateDiskUsage(start, end []byte) (uint64, error) { return 0, errCorruptIndexEntry } endIdxBlock, err := r.readBlock(context.Background(), - endIdxBH.BlockHandle, nil /* transform */, nil /* readHandle */, nil /* stats */) + endIdxBH.BlockHandle, nil /* transform */, nil /* readHandle */, nil /* stats */, nil /* buffer pool */) if err != nil { return 0, err } @@ -3432,299 +1173,12 @@ func NewReader(f objstorage.Readable, o ReaderOptions, extraOpts ...ReaderOption if r.err != nil { return nil, r.Close() } + return r, nil } -// Layout describes the block organization of an sstable. -type Layout struct { - // NOTE: changes to fields in this struct should also be reflected in - // ValidateBlockChecksums, which validates a static list of BlockHandles - // referenced in this struct. - - Data []BlockHandleWithProperties - Index []BlockHandle - TopIndex BlockHandle - Filter BlockHandle - RangeDel BlockHandle - RangeKey BlockHandle - ValueBlock []BlockHandle - ValueIndex BlockHandle - Properties BlockHandle - MetaIndex BlockHandle - Footer BlockHandle - Format TableFormat -} - -// Describe returns a description of the layout. If the verbose parameter is -// true, details of the structure of each block are returned as well. -func (l *Layout) Describe( - w io.Writer, verbose bool, r *Reader, fmtRecord func(key *base.InternalKey, value []byte), -) { - type block struct { - BlockHandle - name string - } - var blocks []block - - for i := range l.Data { - blocks = append(blocks, block{l.Data[i].BlockHandle, "data"}) - } - for i := range l.Index { - blocks = append(blocks, block{l.Index[i], "index"}) - } - if l.TopIndex.Length != 0 { - blocks = append(blocks, block{l.TopIndex, "top-index"}) - } - if l.Filter.Length != 0 { - blocks = append(blocks, block{l.Filter, "filter"}) - } - if l.RangeDel.Length != 0 { - blocks = append(blocks, block{l.RangeDel, "range-del"}) - } - if l.RangeKey.Length != 0 { - blocks = append(blocks, block{l.RangeKey, "range-key"}) - } - for i := range l.ValueBlock { - blocks = append(blocks, block{l.ValueBlock[i], "value-block"}) - } - if l.ValueIndex.Length != 0 { - blocks = append(blocks, block{l.ValueIndex, "value-index"}) - } - if l.Properties.Length != 0 { - blocks = append(blocks, block{l.Properties, "properties"}) - } - if l.MetaIndex.Length != 0 { - blocks = append(blocks, block{l.MetaIndex, "meta-index"}) - } - if l.Footer.Length != 0 { - if l.Footer.Length == levelDBFooterLen { - blocks = append(blocks, block{l.Footer, "leveldb-footer"}) - } else { - blocks = append(blocks, block{l.Footer, "footer"}) - } - } - - sort.Slice(blocks, func(i, j int) bool { - return blocks[i].Offset < blocks[j].Offset - }) - - for i := range blocks { - b := &blocks[i] - fmt.Fprintf(w, "%10d %s (%d)\n", b.Offset, b.name, b.Length) - - if !verbose { - continue - } - if b.name == "filter" { - continue - } - - if b.name == "footer" || b.name == "leveldb-footer" { - trailer, offset := make([]byte, b.Length), b.Offset - _, _ = r.readable.ReadAt(trailer, int64(offset)) - - if b.name == "footer" { - checksumType := ChecksumType(trailer[0]) - fmt.Fprintf(w, "%10d checksum type: %s\n", offset, checksumType) - trailer, offset = trailer[1:], offset+1 - } - - metaHandle, n := binary.Uvarint(trailer) - metaLen, m := binary.Uvarint(trailer[n:]) - fmt.Fprintf(w, "%10d meta: offset=%d, length=%d\n", offset, metaHandle, metaLen) - trailer, offset = trailer[n+m:], offset+uint64(n+m) - - indexHandle, n := binary.Uvarint(trailer) - indexLen, m := binary.Uvarint(trailer[n:]) - fmt.Fprintf(w, "%10d index: offset=%d, length=%d\n", offset, indexHandle, indexLen) - trailer, offset = trailer[n+m:], offset+uint64(n+m) - - fmt.Fprintf(w, "%10d [padding]\n", offset) - - trailing := 12 - if b.name == "leveldb-footer" { - trailing = 8 - } - - offset += uint64(len(trailer) - trailing) - trailer = trailer[len(trailer)-trailing:] - - if b.name == "footer" { - version := trailer[:4] - fmt.Fprintf(w, "%10d version: %d\n", offset, binary.LittleEndian.Uint32(version)) - trailer, offset = trailer[4:], offset+4 - } - - magicNumber := trailer - fmt.Fprintf(w, "%10d magic number: 0x%x\n", offset, magicNumber) - - continue - } - - h, err := r.readBlock( - context.Background(), b.BlockHandle, nil /* transform */, nil /* readHandle */, nil /* stats */) - if err != nil { - fmt.Fprintf(w, " [err: %s]\n", err) - continue - } - - getRestart := func(data []byte, restarts, i int32) int32 { - return decodeRestart(data[restarts+4*i:]) - } - - formatIsRestart := func(data []byte, restarts, numRestarts, offset int32) { - i := sort.Search(int(numRestarts), func(i int) bool { - return getRestart(data, restarts, int32(i)) >= offset - }) - if i < int(numRestarts) && getRestart(data, restarts, int32(i)) == offset { - fmt.Fprintf(w, " [restart]\n") - } else { - fmt.Fprintf(w, "\n") - } - } - - formatRestarts := func(data []byte, restarts, numRestarts int32) { - for i := int32(0); i < numRestarts; i++ { - offset := getRestart(data, restarts, i) - fmt.Fprintf(w, "%10d [restart %d]\n", - b.Offset+uint64(restarts+4*i), b.Offset+uint64(offset)) - } - } - - formatTrailer := func() { - trailer := make([]byte, blockTrailerLen) - offset := int64(b.Offset + b.Length) - _, _ = r.readable.ReadAt(trailer, offset) - bt := blockType(trailer[0]) - checksum := binary.LittleEndian.Uint32(trailer[1:]) - fmt.Fprintf(w, "%10d [trailer compression=%s checksum=0x%04x]\n", offset, bt, checksum) - } - - var lastKey InternalKey - switch b.name { - case "data", "range-del", "range-key": - iter, _ := newBlockIter(r.Compare, h.Get()) - for key, value := iter.First(); key != nil; key, value = iter.Next() { - ptr := unsafe.Pointer(uintptr(iter.ptr) + uintptr(iter.offset)) - shared, ptr := decodeVarint(ptr) - unshared, ptr := decodeVarint(ptr) - value2, _ := decodeVarint(ptr) - - total := iter.nextOffset - iter.offset - // The format of the numbers in the record line is: - // - // ( = [] + + ) - // - // is the total number of bytes for the record. - // is the size of the 3 varint encoded integers for , - // , and . - // is the number of key bytes shared with the previous key. - // is the number of unshared key bytes. - // is the number of value bytes. - fmt.Fprintf(w, "%10d record (%d = %d [%d] + %d + %d)", - b.Offset+uint64(iter.offset), total, - total-int32(unshared+value2), shared, unshared, value2) - formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset) - if fmtRecord != nil { - fmt.Fprintf(w, " ") - if l.Format != TableFormatPebblev3 { - fmtRecord(key, value.InPlaceValue()) - } else { - // InPlaceValue() will succeed even for data blocks where the - // actual value is in a different location, since this value was - // fetched from a blockIter which does not know about value - // blocks. - v := value.InPlaceValue() - if base.TrailerKind(key.Trailer) != InternalKeyKindSet { - fmtRecord(key, v) - } else if !isValueHandle(valuePrefix(v[0])) { - fmtRecord(key, v[1:]) - } else { - vh := decodeValueHandle(v[1:]) - fmtRecord(key, []byte(fmt.Sprintf("value handle %+v", vh))) - } - } - } - - if base.InternalCompare(r.Compare, lastKey, *key) >= 0 { - fmt.Fprintf(w, " WARNING: OUT OF ORDER KEYS!\n") - } - lastKey.Trailer = key.Trailer - lastKey.UserKey = append(lastKey.UserKey[:0], key.UserKey...) - } - formatRestarts(iter.data, iter.restarts, iter.numRestarts) - formatTrailer() - case "index", "top-index": - iter, _ := newBlockIter(r.Compare, h.Get()) - for key, value := iter.First(); key != nil; key, value = iter.Next() { - bh, err := decodeBlockHandleWithProperties(value.InPlaceValue()) - if err != nil { - fmt.Fprintf(w, "%10d [err: %s]\n", b.Offset+uint64(iter.offset), err) - continue - } - fmt.Fprintf(w, "%10d block:%d/%d", - b.Offset+uint64(iter.offset), bh.Offset, bh.Length) - formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset) - } - formatRestarts(iter.data, iter.restarts, iter.numRestarts) - formatTrailer() - case "properties": - iter, _ := newRawBlockIter(r.Compare, h.Get()) - for valid := iter.First(); valid; valid = iter.Next() { - fmt.Fprintf(w, "%10d %s (%d)", - b.Offset+uint64(iter.offset), iter.Key().UserKey, iter.nextOffset-iter.offset) - formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset) - } - formatRestarts(iter.data, iter.restarts, iter.numRestarts) - formatTrailer() - case "meta-index": - iter, _ := newRawBlockIter(r.Compare, h.Get()) - for valid := iter.First(); valid; valid = iter.Next() { - value := iter.Value() - var bh BlockHandle - var n int - var vbih valueBlocksIndexHandle - isValueBlocksIndexHandle := false - if bytes.Equal(iter.Key().UserKey, []byte(metaValueIndexName)) { - vbih, n, err = decodeValueBlocksIndexHandle(value) - bh = vbih.h - isValueBlocksIndexHandle = true - } else { - bh, n = decodeBlockHandle(value) - } - if n == 0 || n != len(value) { - fmt.Fprintf(w, "%10d [err: %s]\n", b.Offset+uint64(iter.offset), err) - continue - } - var vbihStr string - if isValueBlocksIndexHandle { - vbihStr = fmt.Sprintf(" value-blocks-index-lengths: %d(num), %d(offset), %d(length)", - vbih.blockNumByteLength, vbih.blockOffsetByteLength, vbih.blockLengthByteLength) - } - fmt.Fprintf(w, "%10d %s block:%d/%d%s", - b.Offset+uint64(iter.offset), iter.Key().UserKey, - bh.Offset, bh.Length, vbihStr) - formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset) - } - formatRestarts(iter.data, iter.restarts, iter.numRestarts) - formatTrailer() - case "value-block": - // We don't peer into the value-block since it can't be interpreted - // without the valueHandles. - case "value-index": - // We have already read the value-index to construct the list of - // value-blocks, so no need to do it again. - } - - h.Release() - } - - last := blocks[len(blocks)-1] - fmt.Fprintf(w, "%10d EOF\n", last.Offset+last.Length) -} - -// ReadableFile describes the smallest subset of objstorage.Readable that is -// required for reading SSTs. +// ReadableFile describes the smallest subset of vfs.File that is required for +// reading SSTs. type ReadableFile interface { io.ReaderAt io.Closer @@ -3738,28 +1192,43 @@ func NewSimpleReadable(r ReadableFile) (objstorage.Readable, error) { if err != nil { return nil, err } - return &simpleReadable{ - ReadableFile: r, - size: info.Size(), - rh: objstorage.MakeNoopReadHandle(r), - }, nil + res := &simpleReadable{ + f: r, + size: info.Size(), + } + res.rh = objstorage.MakeNoopReadHandle(res) + return res, nil } // simpleReadable wraps a ReadableFile to implement objstorage.Readable. type simpleReadable struct { - ReadableFile + f ReadableFile size int64 rh objstorage.NoopReadHandle } var _ objstorage.Readable = (*simpleReadable)(nil) +// ReadAt is part of the objstorage.Readable interface. +func (s *simpleReadable) ReadAt(_ context.Context, p []byte, off int64) error { + n, err := s.f.ReadAt(p, off) + if invariants.Enabled && err == nil && n != len(p) { + panic("short read") + } + return err +} + +// Close is part of the objstorage.Readable interface. +func (s *simpleReadable) Close() error { + return s.f.Close() +} + // Size is part of the objstorage.Readable interface. func (s *simpleReadable) Size() int64 { return s.size } // NewReaddHandle is part of the objstorage.Readable interface. -func (s *simpleReadable) NewReadHandle() objstorage.ReadHandle { +func (s *simpleReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle { return &s.rh } diff --git a/vendor/github.com/cockroachdb/pebble/sstable/reader_iter.go b/vendor/github.com/cockroachdb/pebble/sstable/reader_iter.go new file mode 100644 index 0000000..2b5a267 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/sstable/reader_iter.go @@ -0,0 +1,291 @@ +// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package sstable + +import ( + "fmt" + "os" + "sync" + + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" +) + +// Iterator iterates over an entire table of data. +type Iterator interface { + base.InternalIterator + + // NextPrefix implements (base.InternalIterator).NextPrefix. + NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) + + // MaybeFilteredKeys may be called when an iterator is exhausted to indicate + // whether or not the last positioning method may have skipped any keys due + // to block-property filters. This is used by the Pebble levelIter to + // control when an iterator steps to the next sstable. + // + // MaybeFilteredKeys may always return false positives, that is it may + // return true when no keys were filtered. It should only be called when the + // iterator is exhausted. It must never return false negatives when the + // iterator is exhausted. + MaybeFilteredKeys() bool + + SetCloseHook(fn func(i Iterator) error) +} + +// Iterator positioning optimizations and singleLevelIterator and +// twoLevelIterator: +// +// An iterator is absolute positioned using one of the Seek or First or Last +// calls. After absolute positioning, there can be relative positioning done +// by stepping using Prev or Next. +// +// We implement optimizations below where an absolute positioning call can in +// some cases use the current position to do less work. To understand these, +// we first define some terms. An iterator is bounds-exhausted if the bounds +// (upper of lower) have been reached. An iterator is data-exhausted if it has +// the reached the end of the data (forward or reverse) in the sstable. A +// singleLevelIterator only knows a local-data-exhausted property since when +// it is used as part of a twoLevelIterator, the twoLevelIterator can step to +// the next lower-level index block. +// +// The bounds-exhausted property is tracked by +// singleLevelIterator.exhaustedBounds being +1 (upper bound reached) or -1 +// (lower bound reached). The same field is reused by twoLevelIterator. Either +// may notice the exhaustion of the bound and set it. Note that if +// singleLevelIterator sets this property, it is not a local property (since +// the bound has been reached regardless of whether this is in the context of +// the twoLevelIterator or not). +// +// The data-exhausted property is tracked in a more subtle manner. We define +// two predicates: +// - partial-local-data-exhausted (PLDE): +// i.data.isDataInvalidated() || !i.data.valid() +// - partial-global-data-exhausted (PGDE): +// i.index.isDataInvalidated() || !i.index.valid() || i.data.isDataInvalidated() || +// !i.data.valid() +// +// PLDE is defined for a singleLevelIterator. PGDE is defined for a +// twoLevelIterator. Oddly, in our code below the singleLevelIterator does not +// know when it is part of a twoLevelIterator so it does not know when its +// property is local or global. +// +// Now to define data-exhausted: +// - Prerequisite: we must know that the iterator has been positioned and +// i.err is nil. +// - bounds-exhausted must not be true: +// If bounds-exhausted is true, we have incomplete knowledge of +// data-exhausted since PLDE or PGDE could be true because we could have +// chosen not to load index block or data block and figured out that the +// bound is exhausted (due to block property filters filtering out index and +// data blocks and going past the bound on the top level index block). Note +// that if we tried to separate out the BPF case from others we could +// develop more knowledge here. +// - PGDE is true for twoLevelIterator. PLDE is true if it is a standalone +// singleLevelIterator. !PLDE or !PGDE of course imply that data-exhausted +// is not true. +// +// An implication of the above is that if we are going to somehow utilize +// knowledge of data-exhausted in an optimization, we must not forget the +// existing value of bounds-exhausted since by forgetting the latter we can +// erroneously think that data-exhausted is true. Bug #2036 was due to this +// forgetting. +// +// Now to the two categories of optimizations we currently have: +// - Monotonic bounds optimization that reuse prior iterator position when +// doing seek: These only work with !data-exhausted. We could choose to make +// these work with data-exhausted but have not bothered because in the +// context of a DB if data-exhausted were true, the DB would move to the +// next file in the level. Note that this behavior of moving to the next +// file is not necessarily true for L0 files, so there could be some benefit +// in the future in this optimization. See the WARNING-data-exhausted +// comments if trying to optimize this in the future. +// - TrySeekUsingNext optimizations: these work regardless of exhaustion +// state. +// +// Implementation detail: In the code PLDE only checks that +// i.data.isDataInvalidated(). This narrower check is safe, since this is a +// subset of the set expressed by the OR expression. Also, it is not a +// de-optimization since whenever we exhaust the iterator we explicitly call +// i.data.invalidate(). PGDE checks i.index.isDataInvalidated() && +// i.data.isDataInvalidated(). Again, this narrower check is safe, and not a +// de-optimization since whenever we exhaust the iterator we explicitly call +// i.index.invalidate() and i.data.invalidate(). The && is questionable -- for +// now this is a bit of defensive code. We should seriously consider removing +// it, since defensive code suggests we are not confident about our invariants +// (and if we are not confident, we need more invariant assertions, not +// defensive code). +// +// TODO(sumeer): remove the aforementioned defensive code. + +var singleLevelIterPool = sync.Pool{ + New: func() interface{} { + i := &singleLevelIterator{} + // Note: this is a no-op if invariants are disabled or race is enabled. + invariants.SetFinalizer(i, checkSingleLevelIterator) + return i + }, +} + +var twoLevelIterPool = sync.Pool{ + New: func() interface{} { + i := &twoLevelIterator{} + // Note: this is a no-op if invariants are disabled or race is enabled. + invariants.SetFinalizer(i, checkTwoLevelIterator) + return i + }, +} + +// TODO(jackson): rangedel fragmentBlockIters can't be pooled because of some +// code paths that double Close the iters. Fix the double close and pool the +// *fragmentBlockIter type directly. + +var rangeKeyFragmentBlockIterPool = sync.Pool{ + New: func() interface{} { + i := &rangeKeyFragmentBlockIter{} + // Note: this is a no-op if invariants are disabled or race is enabled. + invariants.SetFinalizer(i, checkRangeKeyFragmentBlockIterator) + return i + }, +} + +func checkSingleLevelIterator(obj interface{}) { + i := obj.(*singleLevelIterator) + if p := i.data.handle.Get(); p != nil { + fmt.Fprintf(os.Stderr, "singleLevelIterator.data.handle is not nil: %p\n", p) + os.Exit(1) + } + if p := i.index.handle.Get(); p != nil { + fmt.Fprintf(os.Stderr, "singleLevelIterator.index.handle is not nil: %p\n", p) + os.Exit(1) + } +} + +func checkTwoLevelIterator(obj interface{}) { + i := obj.(*twoLevelIterator) + if p := i.data.handle.Get(); p != nil { + fmt.Fprintf(os.Stderr, "singleLevelIterator.data.handle is not nil: %p\n", p) + os.Exit(1) + } + if p := i.index.handle.Get(); p != nil { + fmt.Fprintf(os.Stderr, "singleLevelIterator.index.handle is not nil: %p\n", p) + os.Exit(1) + } +} + +func checkRangeKeyFragmentBlockIterator(obj interface{}) { + i := obj.(*rangeKeyFragmentBlockIter) + if p := i.blockIter.handle.Get(); p != nil { + fmt.Fprintf(os.Stderr, "fragmentBlockIter.blockIter.handle is not nil: %p\n", p) + os.Exit(1) + } +} + +// compactionIterator is similar to Iterator but it increments the number of +// bytes that have been iterated through. +type compactionIterator struct { + *singleLevelIterator + bytesIterated *uint64 + prevOffset uint64 +} + +// compactionIterator implements the base.InternalIterator interface. +var _ base.InternalIterator = (*compactionIterator)(nil) + +func (i *compactionIterator) String() string { + if i.vState != nil { + return i.vState.fileNum.String() + } + return i.reader.fileNum.String() +} + +func (i *compactionIterator) SeekGE( + key []byte, flags base.SeekGEFlags, +) (*InternalKey, base.LazyValue) { + panic("pebble: SeekGE unimplemented") +} + +func (i *compactionIterator) SeekPrefixGE( + prefix, key []byte, flags base.SeekGEFlags, +) (*base.InternalKey, base.LazyValue) { + panic("pebble: SeekPrefixGE unimplemented") +} + +func (i *compactionIterator) SeekLT( + key []byte, flags base.SeekLTFlags, +) (*InternalKey, base.LazyValue) { + panic("pebble: SeekLT unimplemented") +} + +func (i *compactionIterator) First() (*InternalKey, base.LazyValue) { + i.err = nil // clear cached iteration error + return i.skipForward(i.singleLevelIterator.First()) +} + +func (i *compactionIterator) Last() (*InternalKey, base.LazyValue) { + panic("pebble: Last unimplemented") +} + +// Note: compactionIterator.Next mirrors the implementation of Iterator.Next +// due to performance. Keep the two in sync. +func (i *compactionIterator) Next() (*InternalKey, base.LazyValue) { + if i.err != nil { + return nil, base.LazyValue{} + } + return i.skipForward(i.data.Next()) +} + +func (i *compactionIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { + panic("pebble: NextPrefix unimplemented") +} + +func (i *compactionIterator) Prev() (*InternalKey, base.LazyValue) { + panic("pebble: Prev unimplemented") +} + +func (i *compactionIterator) skipForward( + key *InternalKey, val base.LazyValue, +) (*InternalKey, base.LazyValue) { + if key == nil { + for { + if key, _ := i.index.Next(); key == nil { + break + } + result := i.loadBlock(+1) + if result != loadBlockOK { + if i.err != nil { + break + } + switch result { + case loadBlockFailed: + // We checked that i.index was at a valid entry, so + // loadBlockFailed could not have happened due to to i.index + // being exhausted, and must be due to an error. + panic("loadBlock should not have failed with no error") + case loadBlockIrrelevant: + panic("compactionIter should not be using block intervals for skipping") + default: + panic(fmt.Sprintf("unexpected case %d", result)) + } + } + // result == loadBlockOK + if key, val = i.data.First(); key != nil { + break + } + } + } + + curOffset := i.recordOffset() + *i.bytesIterated += uint64(curOffset - i.prevOffset) + i.prevOffset = curOffset + + if i.vState != nil && key != nil { + cmp := i.cmp(key.UserKey, i.vState.upper.UserKey) + if cmp > 0 || (i.vState.upper.IsExclusiveSentinel() && cmp == 0) { + return nil, base.LazyValue{} + } + } + + return key, val +} diff --git a/vendor/github.com/cockroachdb/pebble/sstable/reader_iter_single_lvl.go b/vendor/github.com/cockroachdb/pebble/sstable/reader_iter_single_lvl.go new file mode 100644 index 0000000..8b094cd --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/sstable/reader_iter_single_lvl.go @@ -0,0 +1,1391 @@ +// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package sstable + +import ( + "context" + "fmt" + "unsafe" + + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing" +) + +// singleLevelIterator iterates over an entire table of data. To seek for a given +// key, it first looks in the index for the block that contains that key, and then +// looks inside that block. +type singleLevelIterator struct { + ctx context.Context + cmp Compare + // Global lower/upper bound for the iterator. + lower []byte + upper []byte + bpfs *BlockPropertiesFilterer + // Per-block lower/upper bound. Nil if the bound does not apply to the block + // because we determined the block lies completely within the bound. + blockLower []byte + blockUpper []byte + reader *Reader + // vState will be set iff the iterator is constructed for virtual sstable + // iteration. + vState *virtualState + // endKeyInclusive is set to force the iterator to treat the upper field as + // inclusive while iterating instead of exclusive. + endKeyInclusive bool + index blockIter + data blockIter + dataRH objstorage.ReadHandle + dataRHPrealloc objstorageprovider.PreallocatedReadHandle + // dataBH refers to the last data block that the iterator considered + // loading. It may not actually have loaded the block, due to an error or + // because it was considered irrelevant. + dataBH BlockHandle + vbReader *valueBlockReader + // vbRH is the read handle for value blocks, which are in a different + // part of the sstable than data blocks. + vbRH objstorage.ReadHandle + vbRHPrealloc objstorageprovider.PreallocatedReadHandle + err error + closeHook func(i Iterator) error + stats *base.InternalIteratorStats + bufferPool *BufferPool + + // boundsCmp and positionedUsingLatestBounds are for optimizing iteration + // that uses multiple adjacent bounds. The seek after setting a new bound + // can use the fact that the iterator is either within the previous bounds + // or exactly one key before or after the bounds. If the new bounds is + // after/before the previous bounds, and we are already positioned at a + // block that is relevant for the new bounds, we can try to first position + // using Next/Prev (repeatedly) instead of doing a more expensive seek. + // + // When there are wide files at higher levels that match the bounds + // but don't have any data for the bound, we will already be + // positioned at the key beyond the bounds and won't need to do much + // work -- given that most data is in L6, such files are likely to + // dominate the performance of the mergingIter, and may be the main + // benefit of this performance optimization (of course it also helps + // when the file that has the data has successive seeks that stay in + // the same block). + // + // Specifically, boundsCmp captures the relationship between the previous + // and current bounds, if the iterator had been positioned after setting + // the previous bounds. If it was not positioned, i.e., Seek/First/Last + // were not called, we don't know where it is positioned and cannot + // optimize. + // + // Example: Bounds moving forward, and iterator exhausted in forward direction. + // bounds = [f, h), ^ shows block iterator position + // file contents [ a b c d e f g h i j k ] + // ^ + // new bounds = [j, k). Since positionedUsingLatestBounds=true, boundsCmp is + // set to +1. SeekGE(j) can use next (the optimization also requires that j + // is within the block, but that is not for correctness, but to limit the + // optimization to when it will actually be an optimization). + // + // Example: Bounds moving forward. + // bounds = [f, h), ^ shows block iterator position + // file contents [ a b c d e f g h i j k ] + // ^ + // new bounds = [j, k). Since positionedUsingLatestBounds=true, boundsCmp is + // set to +1. SeekGE(j) can use next. + // + // Example: Bounds moving forward, but iterator not positioned using previous + // bounds. + // bounds = [f, h), ^ shows block iterator position + // file contents [ a b c d e f g h i j k ] + // ^ + // new bounds = [i, j). Iterator is at j since it was never positioned using + // [f, h). So positionedUsingLatestBounds=false, and boundsCmp is set to 0. + // SeekGE(i) will not use next. + // + // Example: Bounds moving forward and sparse file + // bounds = [f, h), ^ shows block iterator position + // file contents [ a z ] + // ^ + // new bounds = [j, k). Since positionedUsingLatestBounds=true, boundsCmp is + // set to +1. SeekGE(j) notices that the iterator is already past j and does + // not need to do anything. + // + // Similar examples can be constructed for backward iteration. + // + // This notion of exactly one key before or after the bounds is not quite + // true when block properties are used to ignore blocks. In that case we + // can't stop precisely at the first block that is past the bounds since + // we are using the index entries to enforce the bounds. + // + // e.g. 3 blocks with keys [b, c] [f, g], [i, j, k] with index entries d, + // h, l. And let the lower bound be k, and we are reverse iterating. If + // the block [i, j, k] is ignored due to the block interval annotations we + // do need to move the index to block [f, g] since the index entry for the + // [i, j, k] block is l which is not less than the lower bound of k. So we + // have passed the entries i, j. + // + // This behavior is harmless since the block property filters are fixed + // for the lifetime of the iterator so i, j are irrelevant. In addition, + // the current code will not load the [f, g] block, so the seek + // optimization that attempts to use Next/Prev do not apply anyway. + boundsCmp int + positionedUsingLatestBounds bool + + // exhaustedBounds represents whether the iterator is exhausted for + // iteration by reaching the upper or lower bound. +1 when exhausted + // the upper bound, -1 when exhausted the lower bound, and 0 when + // neither. exhaustedBounds is also used for the TrySeekUsingNext + // optimization in twoLevelIterator and singleLevelIterator. Care should be + // taken in setting this in twoLevelIterator before calling into + // singleLevelIterator, given that these two iterators share this field. + exhaustedBounds int8 + + // maybeFilteredKeysSingleLevel indicates whether the last iterator + // positioning operation may have skipped any data blocks due to + // block-property filters when positioning the index. + maybeFilteredKeysSingleLevel bool + + // useFilter specifies whether the filter block in this sstable, if present, + // should be used for prefix seeks or not. In some cases it is beneficial + // to skip a filter block even if it exists (eg. if probability of a match + // is high). + useFilter bool + lastBloomFilterMatched bool + + hideObsoletePoints bool +} + +// singleLevelIterator implements the base.InternalIterator interface. +var _ base.InternalIterator = (*singleLevelIterator)(nil) + +// init initializes a singleLevelIterator for reading from the table. It is +// synonmous with Reader.NewIter, but allows for reusing of the iterator +// between different Readers. +// +// Note that lower, upper passed into init has nothing to do with virtual sstable +// bounds. If the virtualState passed in is not nil, then virtual sstable bounds +// will be enforced. +func (i *singleLevelIterator) init( + ctx context.Context, + r *Reader, + v *virtualState, + lower, upper []byte, + filterer *BlockPropertiesFilterer, + useFilter, hideObsoletePoints bool, + stats *base.InternalIteratorStats, + rp ReaderProvider, + bufferPool *BufferPool, +) error { + if r.err != nil { + return r.err + } + indexH, err := r.readIndex(ctx, stats) + if err != nil { + return err + } + if v != nil { + i.vState = v + i.endKeyInclusive, lower, upper = v.constrainBounds(lower, upper, false /* endInclusive */) + } + + i.ctx = ctx + i.lower = lower + i.upper = upper + i.bpfs = filterer + i.useFilter = useFilter + i.reader = r + i.cmp = r.Compare + i.stats = stats + i.hideObsoletePoints = hideObsoletePoints + i.bufferPool = bufferPool + err = i.index.initHandle(i.cmp, indexH, r.Properties.GlobalSeqNum, false) + if err != nil { + // blockIter.Close releases indexH and always returns a nil error + _ = i.index.Close() + return err + } + i.dataRH = objstorageprovider.UsePreallocatedReadHandle(ctx, r.readable, &i.dataRHPrealloc) + if r.tableFormat >= TableFormatPebblev3 { + if r.Properties.NumValueBlocks > 0 { + // NB: we cannot avoid this ~248 byte allocation, since valueBlockReader + // can outlive the singleLevelIterator due to be being embedded in a + // LazyValue. This consumes ~2% in microbenchmark CPU profiles, but we + // should only optimize this if it shows up as significant in end-to-end + // CockroachDB benchmarks, since it is tricky to do so. One possibility + // is that if many sstable iterators only get positioned at latest + // versions of keys, and therefore never expose a LazyValue that is + // separated to their callers, they can put this valueBlockReader into a + // sync.Pool. + i.vbReader = &valueBlockReader{ + ctx: ctx, + bpOpen: i, + rp: rp, + vbih: r.valueBIH, + stats: stats, + } + i.data.lazyValueHandling.vbr = i.vbReader + i.vbRH = objstorageprovider.UsePreallocatedReadHandle(ctx, r.readable, &i.vbRHPrealloc) + } + i.data.lazyValueHandling.hasValuePrefix = true + } + return nil +} + +// Helper function to check if keys returned from iterator are within global and virtual bounds. +func (i *singleLevelIterator) maybeVerifyKey( + iKey *InternalKey, val base.LazyValue, +) (*InternalKey, base.LazyValue) { + // maybeVerify key is only used for virtual sstable iterators. + if invariants.Enabled && i.vState != nil && iKey != nil { + key := iKey.UserKey + + uc, vuc := i.cmp(key, i.upper), i.cmp(key, i.vState.upper.UserKey) + lc, vlc := i.cmp(key, i.lower), i.cmp(key, i.vState.lower.UserKey) + + if (i.vState.upper.IsExclusiveSentinel() && vuc == 0) || (!i.endKeyInclusive && uc == 0) || uc > 0 || vuc > 0 || lc < 0 || vlc < 0 { + panic(fmt.Sprintf("key: %s out of bounds of singleLevelIterator", key)) + } + } + return iKey, val +} + +// setupForCompaction sets up the singleLevelIterator for use with compactionIter. +// Currently, it skips readahead ramp-up. It should be called after init is called. +func (i *singleLevelIterator) setupForCompaction() { + i.dataRH.SetupForCompaction() + if i.vbRH != nil { + i.vbRH.SetupForCompaction() + } +} + +func (i *singleLevelIterator) resetForReuse() singleLevelIterator { + return singleLevelIterator{ + index: i.index.resetForReuse(), + data: i.data.resetForReuse(), + } +} + +func (i *singleLevelIterator) initBounds() { + // Trim the iteration bounds for the current block. We don't have to check + // the bounds on each iteration if the block is entirely contained within the + // iteration bounds. + i.blockLower = i.lower + if i.blockLower != nil { + key, _ := i.data.First() + if key != nil && i.cmp(i.blockLower, key.UserKey) < 0 { + // The lower-bound is less than the first key in the block. No need + // to check the lower-bound again for this block. + i.blockLower = nil + } + } + i.blockUpper = i.upper + if i.blockUpper != nil && i.cmp(i.blockUpper, i.index.Key().UserKey) > 0 { + // The upper-bound is greater than the index key which itself is greater + // than or equal to every key in the block. No need to check the + // upper-bound again for this block. Even if blockUpper is inclusive + // because of upper being inclusive, we can still safely set blockUpper + // to nil here. + // + // TODO(bananabrick): We could also set blockUpper to nil for the >= + // case, if blockUpper is inclusive. + i.blockUpper = nil + } +} + +// Deterministic disabling of the bounds-based optimization that avoids seeking. +// Uses the iterator pointer, since we want diversity in iterator behavior for +// the same SetBounds call. Used for tests. +func disableBoundsOpt(bound []byte, ptr uintptr) bool { + // Fibonacci hash https://probablydance.com/2018/06/16/fibonacci-hashing-the-optimization-that-the-world-forgot-or-a-better-alternative-to-integer-modulo/ + simpleHash := (11400714819323198485 * uint64(ptr)) >> 63 + return bound[len(bound)-1]&byte(1) == 0 && simpleHash == 0 +} + +// ensureBoundsOptDeterminism provides a facility for disabling of the bounds +// optimizations performed by disableBoundsOpt for tests that require +// deterministic iterator behavior. Some unit tests examine internal iterator +// state and require this behavior to be deterministic. +var ensureBoundsOptDeterminism bool + +// SetBounds implements internalIterator.SetBounds, as documented in the pebble +// package. Note that the upper field is exclusive. +func (i *singleLevelIterator) SetBounds(lower, upper []byte) { + i.boundsCmp = 0 + if i.vState != nil { + // If the reader is constructed for a virtual sstable, then we must + // constrain the bounds of the reader. For physical sstables, the bounds + // can be wider than the actual sstable's bounds because we won't + // accidentally expose additional keys as there are no additional keys. + i.endKeyInclusive, lower, upper = i.vState.constrainBounds( + lower, upper, false, + ) + } else { + // TODO(bananabrick): Figure out the logic here to enable the boundsCmp + // optimization for virtual sstables. + if i.positionedUsingLatestBounds { + if i.upper != nil && lower != nil && i.cmp(i.upper, lower) <= 0 { + i.boundsCmp = +1 + if invariants.Enabled && !ensureBoundsOptDeterminism && + disableBoundsOpt(lower, uintptr(unsafe.Pointer(i))) { + i.boundsCmp = 0 + } + } else if i.lower != nil && upper != nil && i.cmp(upper, i.lower) <= 0 { + i.boundsCmp = -1 + if invariants.Enabled && !ensureBoundsOptDeterminism && + disableBoundsOpt(upper, uintptr(unsafe.Pointer(i))) { + i.boundsCmp = 0 + } + } + } + } + + i.positionedUsingLatestBounds = false + i.lower = lower + i.upper = upper + i.blockLower = nil + i.blockUpper = nil +} + +// loadBlock loads the block at the current index position and leaves i.data +// unpositioned. If unsuccessful, it sets i.err to any error encountered, which +// may be nil if we have simply exhausted the entire table. +func (i *singleLevelIterator) loadBlock(dir int8) loadBlockResult { + if !i.index.valid() { + // Ensure the data block iterator is invalidated even if loading of the block + // fails. + i.data.invalidate() + return loadBlockFailed + } + // Load the next block. + v := i.index.value() + bhp, err := decodeBlockHandleWithProperties(v.InPlaceValue()) + if i.dataBH == bhp.BlockHandle && i.data.valid() { + // We're already at the data block we want to load. Reset bounds in case + // they changed since the last seek, but don't reload the block from cache + // or disk. + // + // It's safe to leave i.data in its original state here, as all callers to + // loadBlock make an absolute positioning call (i.e. a seek, first, or last) + // to `i.data` right after loadBlock returns loadBlockOK. + i.initBounds() + return loadBlockOK + } + // Ensure the data block iterator is invalidated even if loading of the block + // fails. + i.data.invalidate() + i.dataBH = bhp.BlockHandle + if err != nil { + i.err = errCorruptIndexEntry + return loadBlockFailed + } + if i.bpfs != nil { + intersects, err := i.bpfs.intersects(bhp.Props) + if err != nil { + i.err = errCorruptIndexEntry + return loadBlockFailed + } + if intersects == blockMaybeExcluded { + intersects = i.resolveMaybeExcluded(dir) + } + if intersects == blockExcluded { + i.maybeFilteredKeysSingleLevel = true + return loadBlockIrrelevant + } + // blockIntersects + } + ctx := objiotracing.WithBlockType(i.ctx, objiotracing.DataBlock) + block, err := i.reader.readBlock(ctx, i.dataBH, nil /* transform */, i.dataRH, i.stats, i.bufferPool) + if err != nil { + i.err = err + return loadBlockFailed + } + i.err = i.data.initHandle(i.cmp, block, i.reader.Properties.GlobalSeqNum, i.hideObsoletePoints) + if i.err != nil { + // The block is partially loaded, and we don't want it to appear valid. + i.data.invalidate() + return loadBlockFailed + } + i.initBounds() + return loadBlockOK +} + +// readBlockForVBR implements the blockProviderWhenOpen interface for use by +// the valueBlockReader. +func (i *singleLevelIterator) readBlockForVBR( + ctx context.Context, h BlockHandle, stats *base.InternalIteratorStats, +) (bufferHandle, error) { + ctx = objiotracing.WithBlockType(ctx, objiotracing.ValueBlock) + return i.reader.readBlock(ctx, h, nil, i.vbRH, stats, i.bufferPool) +} + +// resolveMaybeExcluded is invoked when the block-property filterer has found +// that a block is excluded according to its properties but only if its bounds +// fall within the filter's current bounds. This function consults the +// apprioriate bound, depending on the iteration direction, and returns either +// `blockIntersects` or `blockMaybeExcluded`. +func (i *singleLevelIterator) resolveMaybeExcluded(dir int8) intersectsResult { + // TODO(jackson): We could first try comparing to top-level index block's + // key, and if within bounds avoid per-data block key comparisons. + + // This iterator is configured with a bound-limited block property + // filter. The bpf determined this block could be excluded from + // iteration based on the property encoded in the block handle. + // However, we still need to determine if the block is wholly + // contained within the filter's key bounds. + // + // External guarantees ensure all the block's keys are ≥ the + // filter's lower bound during forward iteration, and that all the + // block's keys are < the filter's upper bound during backward + // iteration. We only need to determine if the opposite bound is + // also met. + // + // The index separator in index.Key() provides an inclusive + // upper-bound for the data block's keys, guaranteeing that all its + // keys are ≤ index.Key(). For forward iteration, this is all we + // need. + if dir > 0 { + // Forward iteration. + if i.bpfs.boundLimitedFilter.KeyIsWithinUpperBound(i.index.Key().UserKey) { + return blockExcluded + } + return blockIntersects + } + + // Reverse iteration. + // + // Because we're iterating in the reverse direction, we don't yet have + // enough context available to determine if the block is wholly contained + // within its bounds. This case arises only during backward iteration, + // because of the way the index is structured. + // + // Consider a bound-limited bpf limited to the bounds [b,d), loading the + // block with separator `c`. During reverse iteration, the guarantee that + // all the block's keys are < `d` is externally provided, but no guarantee + // is made on the bpf's lower bound. The separator `c` only provides an + // inclusive upper bound on the block's keys, indicating that the + // corresponding block handle points to a block containing only keys ≤ `c`. + // + // To establish a lower bound, we step the index backwards to read the + // previous block's separator, which provides an inclusive lower bound on + // the original block's keys. Afterwards, we step forward to restore our + // index position. + if peekKey, _ := i.index.Prev(); peekKey == nil { + // The original block points to the first block of this index block. If + // there's a two-level index, it could potentially provide a lower + // bound, but the code refactoring necessary to read it doesn't seem + // worth the payoff. We fall through to loading the block. + } else if i.bpfs.boundLimitedFilter.KeyIsWithinLowerBound(peekKey.UserKey) { + // The lower-bound on the original block falls within the filter's + // bounds, and we can skip the block (after restoring our current index + // position). + _, _ = i.index.Next() + return blockExcluded + } + _, _ = i.index.Next() + return blockIntersects +} + +func (i *singleLevelIterator) initBoundsForAlreadyLoadedBlock() { + if i.data.getFirstUserKey() == nil { + panic("initBoundsForAlreadyLoadedBlock must not be called on empty or corrupted block") + } + i.blockLower = i.lower + if i.blockLower != nil { + firstUserKey := i.data.getFirstUserKey() + if firstUserKey != nil && i.cmp(i.blockLower, firstUserKey) < 0 { + // The lower-bound is less than the first key in the block. No need + // to check the lower-bound again for this block. + i.blockLower = nil + } + } + i.blockUpper = i.upper + if i.blockUpper != nil && i.cmp(i.blockUpper, i.index.Key().UserKey) > 0 { + // The upper-bound is greater than the index key which itself is greater + // than or equal to every key in the block. No need to check the + // upper-bound again for this block. + i.blockUpper = nil + } +} + +// The number of times to call Next/Prev in a block before giving up and seeking. +// The value of 4 is arbitrary. +// TODO(sumeer): experiment with dynamic adjustment based on the history of +// seeks for a particular iterator. +const numStepsBeforeSeek = 4 + +func (i *singleLevelIterator) trySeekGEUsingNextWithinBlock( + key []byte, +) (k *InternalKey, v base.LazyValue, done bool) { + k, v = i.data.Key(), i.data.value() + for j := 0; j < numStepsBeforeSeek; j++ { + curKeyCmp := i.cmp(k.UserKey, key) + if curKeyCmp >= 0 { + if i.blockUpper != nil { + cmp := i.cmp(k.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{}, true + } + } + return k, v, true + } + k, v = i.data.Next() + if k == nil { + break + } + } + return k, v, false +} + +func (i *singleLevelIterator) trySeekLTUsingPrevWithinBlock( + key []byte, +) (k *InternalKey, v base.LazyValue, done bool) { + k, v = i.data.Key(), i.data.value() + for j := 0; j < numStepsBeforeSeek; j++ { + curKeyCmp := i.cmp(k.UserKey, key) + if curKeyCmp < 0 { + if i.blockLower != nil && i.cmp(k.UserKey, i.blockLower) < 0 { + i.exhaustedBounds = -1 + return nil, base.LazyValue{}, true + } + return k, v, true + } + k, v = i.data.Prev() + if k == nil { + break + } + } + return k, v, false +} + +func (i *singleLevelIterator) recordOffset() uint64 { + offset := i.dataBH.Offset + if i.data.valid() { + // - i.dataBH.Length/len(i.data.data) is the compression ratio. If + // uncompressed, this is 1. + // - i.data.nextOffset is the uncompressed position of the current record + // in the block. + // - i.dataBH.Offset is the offset of the block in the sstable before + // decompression. + offset += (uint64(i.data.nextOffset) * i.dataBH.Length) / uint64(len(i.data.data)) + } else { + // Last entry in the block must increment bytes iterated by the size of the block trailer + // and restart points. + offset += i.dataBH.Length + blockTrailerLen + } + return offset +} + +// SeekGE implements internalIterator.SeekGE, as documented in the pebble +// package. Note that SeekGE only checks the upper bound. It is up to the +// caller to ensure that key is greater than or equal to the lower bound. +func (i *singleLevelIterator) SeekGE( + key []byte, flags base.SeekGEFlags, +) (*InternalKey, base.LazyValue) { + if i.vState != nil { + // Callers of SeekGE don't know about virtual sstable bounds, so we may + // have to internally restrict the bounds. + // + // TODO(bananabrick): We can optimize this check away for the level iter + // if necessary. + if i.cmp(key, i.lower) < 0 { + key = i.lower + } + } + + if flags.TrySeekUsingNext() { + // The i.exhaustedBounds comparison indicates that the upper bound was + // reached. The i.data.isDataInvalidated() indicates that the sstable was + // exhausted. + if (i.exhaustedBounds == +1 || i.data.isDataInvalidated()) && i.err == nil { + // Already exhausted, so return nil. + return nil, base.LazyValue{} + } + if i.err != nil { + // The current iterator position cannot be used. + flags = flags.DisableTrySeekUsingNext() + } + // INVARIANT: flags.TrySeekUsingNext() => i.err == nil && + // !i.exhaustedBounds==+1 && !i.data.isDataInvalidated(). That is, + // data-exhausted and bounds-exhausted, as defined earlier, are both + // false. Ths makes it safe to clear out i.exhaustedBounds and i.err + // before calling into seekGEHelper. + } + + i.exhaustedBounds = 0 + i.err = nil // clear cached iteration error + boundsCmp := i.boundsCmp + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + i.positionedUsingLatestBounds = true + return i.seekGEHelper(key, boundsCmp, flags) +} + +// seekGEHelper contains the common functionality for SeekGE and SeekPrefixGE. +func (i *singleLevelIterator) seekGEHelper( + key []byte, boundsCmp int, flags base.SeekGEFlags, +) (*InternalKey, base.LazyValue) { + // Invariant: trySeekUsingNext => !i.data.isDataInvalidated() && i.exhaustedBounds != +1 + + // SeekGE performs various step-instead-of-seeking optimizations: eg enabled + // by trySeekUsingNext, or by monotonically increasing bounds (i.boundsCmp). + // Care must be taken to ensure that when performing these optimizations and + // the iterator becomes exhausted, i.maybeFilteredKeys is set appropriately. + // Consider a previous SeekGE that filtered keys from k until the current + // iterator position. + // + // If the previous SeekGE exhausted the iterator, it's possible keys greater + // than or equal to the current search key were filtered. We must not reuse + // the current iterator position without remembering the previous value of + // maybeFilteredKeys. + + var dontSeekWithinBlock bool + if !i.data.isDataInvalidated() && !i.index.isDataInvalidated() && i.data.valid() && i.index.valid() && + boundsCmp > 0 && i.cmp(key, i.index.Key().UserKey) <= 0 { + // Fast-path: The bounds have moved forward and this SeekGE is + // respecting the lower bound (guaranteed by Iterator). We know that + // the iterator must already be positioned within or just outside the + // previous bounds. Therefore it cannot be positioned at a block (or + // the position within that block) that is ahead of the seek position. + // However it can be positioned at an earlier block. This fast-path to + // use Next() on the block is only applied when we are already at the + // block that the slow-path (the else-clause) would load -- this is + // the motivation for the i.cmp(key, i.index.Key().UserKey) <= 0 + // predicate. + i.initBoundsForAlreadyLoadedBlock() + ikey, val, done := i.trySeekGEUsingNextWithinBlock(key) + if done { + return ikey, val + } + if ikey == nil { + // Done with this block. + dontSeekWithinBlock = true + } + } else { + // Cannot use bounds monotonicity. But may be able to optimize if + // caller claimed externally known invariant represented by + // flags.TrySeekUsingNext(). + if flags.TrySeekUsingNext() { + // seekPrefixGE or SeekGE has already ensured + // !i.data.isDataInvalidated() && i.exhaustedBounds != +1 + currKey := i.data.Key() + value := i.data.value() + less := i.cmp(currKey.UserKey, key) < 0 + // We could be more sophisticated and confirm that the seek + // position is within the current block before applying this + // optimization. But there may be some benefit even if it is in + // the next block, since we can avoid seeking i.index. + for j := 0; less && j < numStepsBeforeSeek; j++ { + currKey, value = i.Next() + if currKey == nil { + return nil, base.LazyValue{} + } + less = i.cmp(currKey.UserKey, key) < 0 + } + if !less { + if i.blockUpper != nil { + cmp := i.cmp(currKey.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + return currKey, value + } + } + + // Slow-path. + // Since we're re-seeking the iterator, the previous value of + // maybeFilteredKeysSingleLevel is irrelevant. If we filter out blocks + // during seeking, loadBlock will set it to true. + i.maybeFilteredKeysSingleLevel = false + + var ikey *InternalKey + if ikey, _ = i.index.SeekGE(key, flags.DisableTrySeekUsingNext()); ikey == nil { + // The target key is greater than any key in the index block. + // Invalidate the block iterator so that a subsequent call to Prev() + // will return the last key in the table. + i.data.invalidate() + return nil, base.LazyValue{} + } + result := i.loadBlock(+1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockIrrelevant { + // Enforce the upper bound here since don't want to bother moving + // to the next block if upper bound is already exceeded. Note that + // the next block starts with keys >= ikey.UserKey since even + // though this is the block separator, the same user key can span + // multiple blocks. If upper is exclusive we use >= below, else + // we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + // Want to skip to the next block. + dontSeekWithinBlock = true + } + } + if !dontSeekWithinBlock { + if ikey, val := i.data.SeekGE(key, flags.DisableTrySeekUsingNext()); ikey != nil { + if i.blockUpper != nil { + cmp := i.cmp(ikey.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + return ikey, val + } + } + return i.skipForward() +} + +// SeekPrefixGE implements internalIterator.SeekPrefixGE, as documented in the +// pebble package. Note that SeekPrefixGE only checks the upper bound. It is up +// to the caller to ensure that key is greater than or equal to the lower bound. +func (i *singleLevelIterator) SeekPrefixGE( + prefix, key []byte, flags base.SeekGEFlags, +) (*base.InternalKey, base.LazyValue) { + if i.vState != nil { + // Callers of SeekPrefixGE aren't aware of virtual sstable bounds, so + // we may have to internally restrict the bounds. + // + // TODO(bananabrick): We can optimize away this check for the level iter + // if necessary. + if i.cmp(key, i.lower) < 0 { + key = i.lower + } + } + return i.seekPrefixGE(prefix, key, flags, i.useFilter) +} + +func (i *singleLevelIterator) seekPrefixGE( + prefix, key []byte, flags base.SeekGEFlags, checkFilter bool, +) (k *InternalKey, value base.LazyValue) { + // NOTE: prefix is only used for bloom filter checking and not later work in + // this method. Hence, we can use the existing iterator position if the last + // SeekPrefixGE did not fail bloom filter matching. + + err := i.err + i.err = nil // clear cached iteration error + if checkFilter && i.reader.tableFilter != nil { + if !i.lastBloomFilterMatched { + // Iterator is not positioned based on last seek. + flags = flags.DisableTrySeekUsingNext() + } + i.lastBloomFilterMatched = false + // Check prefix bloom filter. + var dataH bufferHandle + dataH, i.err = i.reader.readFilter(i.ctx, i.stats) + if i.err != nil { + i.data.invalidate() + return nil, base.LazyValue{} + } + mayContain := i.reader.tableFilter.mayContain(dataH.Get(), prefix) + dataH.Release() + if !mayContain { + // This invalidation may not be necessary for correctness, and may + // be a place to optimize later by reusing the already loaded + // block. It was necessary in earlier versions of the code since + // the caller was allowed to call Next when SeekPrefixGE returned + // nil. This is no longer allowed. + i.data.invalidate() + return nil, base.LazyValue{} + } + i.lastBloomFilterMatched = true + } + if flags.TrySeekUsingNext() { + // The i.exhaustedBounds comparison indicates that the upper bound was + // reached. The i.data.isDataInvalidated() indicates that the sstable was + // exhausted. + if (i.exhaustedBounds == +1 || i.data.isDataInvalidated()) && err == nil { + // Already exhausted, so return nil. + return nil, base.LazyValue{} + } + if err != nil { + // The current iterator position cannot be used. + flags = flags.DisableTrySeekUsingNext() + } + // INVARIANT: flags.TrySeekUsingNext() => err == nil && + // !i.exhaustedBounds==+1 && !i.data.isDataInvalidated(). That is, + // data-exhausted and bounds-exhausted, as defined earlier, are both + // false. Ths makes it safe to clear out i.exhaustedBounds and i.err + // before calling into seekGEHelper. + } + // Bloom filter matches, or skipped, so this method will position the + // iterator. + i.exhaustedBounds = 0 + boundsCmp := i.boundsCmp + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + i.positionedUsingLatestBounds = true + k, value = i.seekGEHelper(key, boundsCmp, flags) + return i.maybeVerifyKey(k, value) +} + +// virtualLast should only be called if i.vReader != nil. +func (i *singleLevelIterator) virtualLast() (*InternalKey, base.LazyValue) { + if i.vState == nil { + panic("pebble: invalid call to virtualLast") + } + + // Seek to the first internal key. + ikey, _ := i.SeekGE(i.upper, base.SeekGEFlagsNone) + if i.endKeyInclusive { + // Let's say the virtual sstable upper bound is c#1, with the keys c#3, c#2, + // c#1, d, e, ... in the sstable. So, the last key in the virtual sstable is + // c#1. We can perform SeekGE(i.upper) and then keep nexting until we find + // the last key with userkey == i.upper. + // + // TODO(bananabrick): Think about how to improve this. If many internal keys + // with the same user key at the upper bound then this could be slow, but + // maybe the odds of having many internal keys with the same user key at the + // upper bound are low. + for ikey != nil && i.cmp(ikey.UserKey, i.upper) == 0 { + ikey, _ = i.Next() + } + return i.Prev() + } + + // We seeked to the first key >= i.upper. + return i.Prev() +} + +// SeekLT implements internalIterator.SeekLT, as documented in the pebble +// package. Note that SeekLT only checks the lower bound. It is up to the +// caller to ensure that key is less than or equal to the upper bound. +func (i *singleLevelIterator) SeekLT( + key []byte, flags base.SeekLTFlags, +) (*InternalKey, base.LazyValue) { + if i.vState != nil { + // Might have to fix upper bound since virtual sstable bounds are not + // known to callers of SeekLT. + // + // TODO(bananabrick): We can optimize away this check for the level iter + // if necessary. + cmp := i.cmp(key, i.upper) + // key == i.upper is fine. We'll do the right thing and return the + // first internal key with user key < key. + if cmp > 0 { + // Return the last key in the virtual sstable. + return i.virtualLast() + } + } + + i.exhaustedBounds = 0 + i.err = nil // clear cached iteration error + boundsCmp := i.boundsCmp + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + + // Seeking operations perform various step-instead-of-seeking optimizations: + // eg by considering monotonically increasing bounds (i.boundsCmp). Care + // must be taken to ensure that when performing these optimizations and the + // iterator becomes exhausted i.maybeFilteredKeysSingleLevel is set + // appropriately. Consider a previous SeekLT that filtered keys from k + // until the current iterator position. + // + // If the previous SeekLT did exhausted the iterator, it's possible keys + // less than the current search key were filtered. We must not reuse the + // current iterator position without remembering the previous value of + // maybeFilteredKeysSingleLevel. + + i.positionedUsingLatestBounds = true + + var dontSeekWithinBlock bool + if !i.data.isDataInvalidated() && !i.index.isDataInvalidated() && i.data.valid() && i.index.valid() && + boundsCmp < 0 && i.cmp(i.data.getFirstUserKey(), key) < 0 { + // Fast-path: The bounds have moved backward, and this SeekLT is + // respecting the upper bound (guaranteed by Iterator). We know that + // the iterator must already be positioned within or just outside the + // previous bounds. Therefore it cannot be positioned at a block (or + // the position within that block) that is behind the seek position. + // However it can be positioned at a later block. This fast-path to + // use Prev() on the block is only applied when we are already at the + // block that can satisfy this seek -- this is the motivation for the + // the i.cmp(i.data.firstKey.UserKey, key) < 0 predicate. + i.initBoundsForAlreadyLoadedBlock() + ikey, val, done := i.trySeekLTUsingPrevWithinBlock(key) + if done { + return ikey, val + } + if ikey == nil { + // Done with this block. + dontSeekWithinBlock = true + } + } else { + // Slow-path. + i.maybeFilteredKeysSingleLevel = false + var ikey *InternalKey + + // NB: If a bound-limited block property filter is configured, it's + // externally ensured that the filter is disabled (through returning + // Intersects=false irrespective of the block props provided) during + // seeks. + if ikey, _ = i.index.SeekGE(key, base.SeekGEFlagsNone); ikey == nil { + ikey, _ = i.index.Last() + if ikey == nil { + return nil, base.LazyValue{} + } + } + // INVARIANT: ikey != nil. + result := i.loadBlock(-1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockIrrelevant { + // Enforce the lower bound here since don't want to bother moving + // to the previous block if lower bound is already exceeded. Note + // that the previous block starts with keys <= ikey.UserKey since + // even though this is the current block's separator, the same + // user key can span multiple blocks. + if i.lower != nil && i.cmp(ikey.UserKey, i.lower) < 0 { + i.exhaustedBounds = -1 + return nil, base.LazyValue{} + } + // Want to skip to the previous block. + dontSeekWithinBlock = true + } + } + if !dontSeekWithinBlock { + if ikey, val := i.data.SeekLT(key, flags); ikey != nil { + if i.blockLower != nil && i.cmp(ikey.UserKey, i.blockLower) < 0 { + i.exhaustedBounds = -1 + return nil, base.LazyValue{} + } + return ikey, val + } + } + // The index contains separator keys which may lie between + // user-keys. Consider the user-keys: + // + // complete + // ---- new block --- + // complexion + // + // If these two keys end one block and start the next, the index key may + // be chosen as "compleu". The SeekGE in the index block will then point + // us to the block containing "complexion". If this happens, we want the + // last key from the previous data block. + return i.maybeVerifyKey(i.skipBackward()) +} + +// First implements internalIterator.First, as documented in the pebble +// package. Note that First only checks the upper bound. It is up to the caller +// to ensure that key is greater than or equal to the lower bound (e.g. via a +// call to SeekGE(lower)). +func (i *singleLevelIterator) First() (*InternalKey, base.LazyValue) { + // If the iterator was created on a virtual sstable, we will SeekGE to the + // lower bound instead of using First, because First does not respect + // bounds. + if i.vState != nil { + return i.SeekGE(i.lower, base.SeekGEFlagsNone) + } + + if i.lower != nil { + panic("singleLevelIterator.First() used despite lower bound") + } + i.positionedUsingLatestBounds = true + i.maybeFilteredKeysSingleLevel = false + + return i.firstInternal() +} + +// firstInternal is a helper used for absolute positioning in a single-level +// index file, or for positioning in the second-level index in a two-level +// index file. For the latter, one cannot make any claims about absolute +// positioning. +func (i *singleLevelIterator) firstInternal() (*InternalKey, base.LazyValue) { + i.exhaustedBounds = 0 + i.err = nil // clear cached iteration error + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + + var ikey *InternalKey + if ikey, _ = i.index.First(); ikey == nil { + i.data.invalidate() + return nil, base.LazyValue{} + } + result := i.loadBlock(+1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockOK { + if ikey, val := i.data.First(); ikey != nil { + if i.blockUpper != nil { + cmp := i.cmp(ikey.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + return ikey, val + } + // Else fall through to skipForward. + } else { + // result == loadBlockIrrelevant. Enforce the upper bound here since + // don't want to bother moving to the next block if upper bound is + // already exceeded. Note that the next block starts with keys >= + // ikey.UserKey since even though this is the block separator, the + // same user key can span multiple blocks. If upper is exclusive we + // use >= below, else we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + // Else fall through to skipForward. + } + + return i.skipForward() +} + +// Last implements internalIterator.Last, as documented in the pebble +// package. Note that Last only checks the lower bound. It is up to the caller +// to ensure that key is less than the upper bound (e.g. via a call to +// SeekLT(upper)) +func (i *singleLevelIterator) Last() (*InternalKey, base.LazyValue) { + if i.vState != nil { + return i.virtualLast() + } + + if i.upper != nil { + panic("singleLevelIterator.Last() used despite upper bound") + } + i.positionedUsingLatestBounds = true + i.maybeFilteredKeysSingleLevel = false + return i.lastInternal() +} + +// lastInternal is a helper used for absolute positioning in a single-level +// index file, or for positioning in the second-level index in a two-level +// index file. For the latter, one cannot make any claims about absolute +// positioning. +func (i *singleLevelIterator) lastInternal() (*InternalKey, base.LazyValue) { + i.exhaustedBounds = 0 + i.err = nil // clear cached iteration error + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + + var ikey *InternalKey + if ikey, _ = i.index.Last(); ikey == nil { + i.data.invalidate() + return nil, base.LazyValue{} + } + result := i.loadBlock(-1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockOK { + if ikey, val := i.data.Last(); ikey != nil { + if i.blockLower != nil && i.cmp(ikey.UserKey, i.blockLower) < 0 { + i.exhaustedBounds = -1 + return nil, base.LazyValue{} + } + return ikey, val + } + // Else fall through to skipBackward. + } else { + // result == loadBlockIrrelevant. Enforce the lower bound here since + // don't want to bother moving to the previous block if lower bound is + // already exceeded. Note that the previous block starts with keys <= + // key.UserKey since even though this is the current block's + // separator, the same user key can span multiple blocks. + if i.lower != nil && i.cmp(ikey.UserKey, i.lower) < 0 { + i.exhaustedBounds = -1 + return nil, base.LazyValue{} + } + } + + return i.skipBackward() +} + +// Next implements internalIterator.Next, as documented in the pebble +// package. +// Note: compactionIterator.Next mirrors the implementation of Iterator.Next +// due to performance. Keep the two in sync. +func (i *singleLevelIterator) Next() (*InternalKey, base.LazyValue) { + if i.exhaustedBounds == +1 { + panic("Next called even though exhausted upper bound") + } + i.exhaustedBounds = 0 + i.maybeFilteredKeysSingleLevel = false + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + + if i.err != nil { + return nil, base.LazyValue{} + } + if key, val := i.data.Next(); key != nil { + if i.blockUpper != nil { + cmp := i.cmp(key.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + return key, val + } + return i.skipForward() +} + +// NextPrefix implements (base.InternalIterator).NextPrefix. +func (i *singleLevelIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { + if i.exhaustedBounds == +1 { + panic("NextPrefix called even though exhausted upper bound") + } + i.exhaustedBounds = 0 + i.maybeFilteredKeysSingleLevel = false + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + if i.err != nil { + return nil, base.LazyValue{} + } + if key, val := i.data.NextPrefix(succKey); key != nil { + if i.blockUpper != nil { + cmp := i.cmp(key.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + return key, val + } + // Did not find prefix in the existing data block. This is the slow-path + // where we effectively seek the iterator. + var ikey *InternalKey + // The key is likely to be in the next data block, so try one step. + if ikey, _ = i.index.Next(); ikey == nil { + // The target key is greater than any key in the index block. + // Invalidate the block iterator so that a subsequent call to Prev() + // will return the last key in the table. + i.data.invalidate() + return nil, base.LazyValue{} + } + if i.cmp(succKey, ikey.UserKey) > 0 { + // Not in the next data block, so seek the index. + if ikey, _ = i.index.SeekGE(succKey, base.SeekGEFlagsNone); ikey == nil { + // The target key is greater than any key in the index block. + // Invalidate the block iterator so that a subsequent call to Prev() + // will return the last key in the table. + i.data.invalidate() + return nil, base.LazyValue{} + } + } + result := i.loadBlock(+1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockIrrelevant { + // Enforce the upper bound here since don't want to bother moving + // to the next block if upper bound is already exceeded. Note that + // the next block starts with keys >= ikey.UserKey since even + // though this is the block separator, the same user key can span + // multiple blocks. If upper is exclusive we use >= below, else we use + // >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + } else if key, val := i.data.SeekGE(succKey, base.SeekGEFlagsNone); key != nil { + if i.blockUpper != nil { + cmp := i.cmp(key.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + return i.maybeVerifyKey(key, val) + } + + return i.skipForward() +} + +// Prev implements internalIterator.Prev, as documented in the pebble +// package. +func (i *singleLevelIterator) Prev() (*InternalKey, base.LazyValue) { + if i.exhaustedBounds == -1 { + panic("Prev called even though exhausted lower bound") + } + i.exhaustedBounds = 0 + i.maybeFilteredKeysSingleLevel = false + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + + if i.err != nil { + return nil, base.LazyValue{} + } + if key, val := i.data.Prev(); key != nil { + if i.blockLower != nil && i.cmp(key.UserKey, i.blockLower) < 0 { + i.exhaustedBounds = -1 + return nil, base.LazyValue{} + } + return key, val + } + return i.skipBackward() +} + +func (i *singleLevelIterator) skipForward() (*InternalKey, base.LazyValue) { + for { + var key *InternalKey + if key, _ = i.index.Next(); key == nil { + i.data.invalidate() + break + } + result := i.loadBlock(+1) + if result != loadBlockOK { + if i.err != nil { + break + } + if result == loadBlockFailed { + // We checked that i.index was at a valid entry, so + // loadBlockFailed could not have happened due to to i.index + // being exhausted, and must be due to an error. + panic("loadBlock should not have failed with no error") + } + // result == loadBlockIrrelevant. Enforce the upper bound here + // since don't want to bother moving to the next block if upper + // bound is already exceeded. Note that the next block starts with + // keys >= key.UserKey since even though this is the block + // separator, the same user key can span multiple blocks. If upper + // is exclusive we use >= below, else we use >. + if i.upper != nil { + cmp := i.cmp(key.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + continue + } + if key, val := i.data.First(); key != nil { + if i.blockUpper != nil { + cmp := i.cmp(key.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + return i.maybeVerifyKey(key, val) + } + } + return nil, base.LazyValue{} +} + +func (i *singleLevelIterator) skipBackward() (*InternalKey, base.LazyValue) { + for { + var key *InternalKey + if key, _ = i.index.Prev(); key == nil { + i.data.invalidate() + break + } + result := i.loadBlock(-1) + if result != loadBlockOK { + if i.err != nil { + break + } + if result == loadBlockFailed { + // We checked that i.index was at a valid entry, so + // loadBlockFailed could not have happened due to to i.index + // being exhausted, and must be due to an error. + panic("loadBlock should not have failed with no error") + } + // result == loadBlockIrrelevant. Enforce the lower bound here + // since don't want to bother moving to the previous block if lower + // bound is already exceeded. Note that the previous block starts with + // keys <= key.UserKey since even though this is the current block's + // separator, the same user key can span multiple blocks. + if i.lower != nil && i.cmp(key.UserKey, i.lower) < 0 { + i.exhaustedBounds = -1 + return nil, base.LazyValue{} + } + continue + } + key, val := i.data.Last() + if key == nil { + return nil, base.LazyValue{} + } + if i.blockLower != nil && i.cmp(key.UserKey, i.blockLower) < 0 { + i.exhaustedBounds = -1 + return nil, base.LazyValue{} + } + return i.maybeVerifyKey(key, val) + } + return nil, base.LazyValue{} +} + +// Error implements internalIterator.Error, as documented in the pebble +// package. +func (i *singleLevelIterator) Error() error { + if err := i.data.Error(); err != nil { + return err + } + return i.err +} + +// MaybeFilteredKeys may be called when an iterator is exhausted to indicate +// whether or not the last positioning method may have skipped any keys due to +// block-property filters. +func (i *singleLevelIterator) MaybeFilteredKeys() bool { + return i.maybeFilteredKeysSingleLevel +} + +// SetCloseHook sets a function that will be called when the iterator is +// closed. +func (i *singleLevelIterator) SetCloseHook(fn func(i Iterator) error) { + i.closeHook = fn +} + +func firstError(err0, err1 error) error { + if err0 != nil { + return err0 + } + return err1 +} + +// Close implements internalIterator.Close, as documented in the pebble +// package. +func (i *singleLevelIterator) Close() error { + var err error + if i.closeHook != nil { + err = firstError(err, i.closeHook(i)) + } + err = firstError(err, i.data.Close()) + err = firstError(err, i.index.Close()) + if i.dataRH != nil { + err = firstError(err, i.dataRH.Close()) + i.dataRH = nil + } + err = firstError(err, i.err) + if i.bpfs != nil { + releaseBlockPropertiesFilterer(i.bpfs) + } + if i.vbReader != nil { + i.vbReader.close() + } + if i.vbRH != nil { + err = firstError(err, i.vbRH.Close()) + i.vbRH = nil + } + *i = i.resetForReuse() + singleLevelIterPool.Put(i) + return err +} + +func (i *singleLevelIterator) String() string { + if i.vState != nil { + return i.vState.fileNum.String() + } + return i.reader.fileNum.String() +} diff --git a/vendor/github.com/cockroachdb/pebble/sstable/reader_iter_two_lvl.go b/vendor/github.com/cockroachdb/pebble/sstable/reader_iter_two_lvl.go new file mode 100644 index 0000000..ed7cb2e --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/sstable/reader_iter_two_lvl.go @@ -0,0 +1,1084 @@ +// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package sstable + +import ( + "context" + "fmt" + + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing" +) + +type twoLevelIterator struct { + singleLevelIterator + // maybeFilteredKeysSingleLevel indicates whether the last iterator + // positioning operation may have skipped any index blocks due to + // block-property filters when positioning the top-level-index. + maybeFilteredKeysTwoLevel bool + topLevelIndex blockIter +} + +// twoLevelIterator implements the base.InternalIterator interface. +var _ base.InternalIterator = (*twoLevelIterator)(nil) + +// loadIndex loads the index block at the current top level index position and +// leaves i.index unpositioned. If unsuccessful, it gets i.err to any error +// encountered, which may be nil if we have simply exhausted the entire table. +// This is used for two level indexes. +func (i *twoLevelIterator) loadIndex(dir int8) loadBlockResult { + // Ensure the index data block iterators are invalidated even if loading of + // the index fails. + i.data.invalidate() + i.index.invalidate() + if !i.topLevelIndex.valid() { + i.index.offset = 0 + i.index.restarts = 0 + return loadBlockFailed + } + v := i.topLevelIndex.value() + bhp, err := decodeBlockHandleWithProperties(v.InPlaceValue()) + if err != nil { + i.err = base.CorruptionErrorf("pebble/table: corrupt top level index entry") + return loadBlockFailed + } + if i.bpfs != nil { + intersects, err := i.bpfs.intersects(bhp.Props) + if err != nil { + i.err = errCorruptIndexEntry + return loadBlockFailed + } + if intersects == blockMaybeExcluded { + intersects = i.resolveMaybeExcluded(dir) + } + if intersects == blockExcluded { + i.maybeFilteredKeysTwoLevel = true + return loadBlockIrrelevant + } + // blockIntersects + } + ctx := objiotracing.WithBlockType(i.ctx, objiotracing.MetadataBlock) + indexBlock, err := i.reader.readBlock(ctx, bhp.BlockHandle, nil /* transform */, nil /* readHandle */, i.stats, i.bufferPool) + if err != nil { + i.err = err + return loadBlockFailed + } + if i.err = i.index.initHandle(i.cmp, indexBlock, i.reader.Properties.GlobalSeqNum, false); i.err == nil { + return loadBlockOK + } + return loadBlockFailed +} + +// resolveMaybeExcluded is invoked when the block-property filterer has found +// that an index block is excluded according to its properties but only if its +// bounds fall within the filter's current bounds. This function consults the +// apprioriate bound, depending on the iteration direction, and returns either +// `blockIntersects` or +// `blockMaybeExcluded`. +func (i *twoLevelIterator) resolveMaybeExcluded(dir int8) intersectsResult { + // This iterator is configured with a bound-limited block property filter. + // The bpf determined this entire index block could be excluded from + // iteration based on the property encoded in the block handle. However, we + // still need to determine if the index block is wholly contained within the + // filter's key bounds. + // + // External guarantees ensure all its data blocks' keys are ≥ the filter's + // lower bound during forward iteration, and that all its data blocks' keys + // are < the filter's upper bound during backward iteration. We only need to + // determine if the opposite bound is also met. + // + // The index separator in topLevelIndex.Key() provides an inclusive + // upper-bound for the index block's keys, guaranteeing that all its keys + // are ≤ topLevelIndex.Key(). For forward iteration, this is all we need. + if dir > 0 { + // Forward iteration. + if i.bpfs.boundLimitedFilter.KeyIsWithinUpperBound(i.topLevelIndex.Key().UserKey) { + return blockExcluded + } + return blockIntersects + } + + // Reverse iteration. + // + // Because we're iterating in the reverse direction, we don't yet have + // enough context available to determine if the block is wholly contained + // within its bounds. This case arises only during backward iteration, + // because of the way the index is structured. + // + // Consider a bound-limited bpf limited to the bounds [b,d), loading the + // block with separator `c`. During reverse iteration, the guarantee that + // all the block's keys are < `d` is externally provided, but no guarantee + // is made on the bpf's lower bound. The separator `c` only provides an + // inclusive upper bound on the block's keys, indicating that the + // corresponding block handle points to a block containing only keys ≤ `c`. + // + // To establish a lower bound, we step the top-level index backwards to read + // the previous block's separator, which provides an inclusive lower bound + // on the original index block's keys. Afterwards, we step forward to + // restore our top-level index position. + if peekKey, _ := i.topLevelIndex.Prev(); peekKey == nil { + // The original block points to the first index block of this table. If + // we knew the lower bound for the entire table, it could provide a + // lower bound, but the code refactoring necessary to read it doesn't + // seem worth the payoff. We fall through to loading the block. + } else if i.bpfs.boundLimitedFilter.KeyIsWithinLowerBound(peekKey.UserKey) { + // The lower-bound on the original index block falls within the filter's + // bounds, and we can skip the block (after restoring our current + // top-level index position). + _, _ = i.topLevelIndex.Next() + return blockExcluded + } + _, _ = i.topLevelIndex.Next() + return blockIntersects +} + +// Note that lower, upper passed into init has nothing to do with virtual sstable +// bounds. If the virtualState passed in is not nil, then virtual sstable bounds +// will be enforced. +func (i *twoLevelIterator) init( + ctx context.Context, + r *Reader, + v *virtualState, + lower, upper []byte, + filterer *BlockPropertiesFilterer, + useFilter, hideObsoletePoints bool, + stats *base.InternalIteratorStats, + rp ReaderProvider, + bufferPool *BufferPool, +) error { + if r.err != nil { + return r.err + } + topLevelIndexH, err := r.readIndex(ctx, stats) + if err != nil { + return err + } + if v != nil { + i.vState = v + // Note that upper is exclusive here. + i.endKeyInclusive, lower, upper = v.constrainBounds(lower, upper, false /* endInclusive */) + } + + i.ctx = ctx + i.lower = lower + i.upper = upper + i.bpfs = filterer + i.useFilter = useFilter + i.reader = r + i.cmp = r.Compare + i.stats = stats + i.hideObsoletePoints = hideObsoletePoints + i.bufferPool = bufferPool + err = i.topLevelIndex.initHandle(i.cmp, topLevelIndexH, r.Properties.GlobalSeqNum, false) + if err != nil { + // blockIter.Close releases topLevelIndexH and always returns a nil error + _ = i.topLevelIndex.Close() + return err + } + i.dataRH = r.readable.NewReadHandle(ctx) + if r.tableFormat >= TableFormatPebblev3 { + if r.Properties.NumValueBlocks > 0 { + i.vbReader = &valueBlockReader{ + ctx: ctx, + bpOpen: i, + rp: rp, + vbih: r.valueBIH, + stats: stats, + } + i.data.lazyValueHandling.vbr = i.vbReader + i.vbRH = r.readable.NewReadHandle(ctx) + } + i.data.lazyValueHandling.hasValuePrefix = true + } + return nil +} + +func (i *twoLevelIterator) String() string { + if i.vState != nil { + return i.vState.fileNum.String() + } + return i.reader.fileNum.String() +} + +// MaybeFilteredKeys may be called when an iterator is exhausted to indicate +// whether or not the last positioning method may have skipped any keys due to +// block-property filters. +func (i *twoLevelIterator) MaybeFilteredKeys() bool { + // While reading sstables with two-level indexes, knowledge of whether we've + // filtered keys is tracked separately for each index level. The + // seek-using-next optimizations have different criteria. We can only reset + // maybeFilteredKeys back to false during a seek when NOT using the + // fast-path that uses the current iterator position. + // + // If either level might have filtered keys to arrive at the current + // iterator position, return MaybeFilteredKeys=true. + return i.maybeFilteredKeysTwoLevel || i.maybeFilteredKeysSingleLevel +} + +// SeekGE implements internalIterator.SeekGE, as documented in the pebble +// package. Note that SeekGE only checks the upper bound. It is up to the +// caller to ensure that key is greater than or equal to the lower bound. +func (i *twoLevelIterator) SeekGE( + key []byte, flags base.SeekGEFlags, +) (*InternalKey, base.LazyValue) { + if i.vState != nil { + // Callers of SeekGE don't know about virtual sstable bounds, so we may + // have to internally restrict the bounds. + // + // TODO(bananabrick): We can optimize away this check for the level iter + // if necessary. + if i.cmp(key, i.lower) < 0 { + key = i.lower + } + } + + err := i.err + i.err = nil // clear cached iteration error + + // The twoLevelIterator could be already exhausted. Utilize that when + // trySeekUsingNext is true. See the comment about data-exhausted, PGDE, and + // bounds-exhausted near the top of the file. + if flags.TrySeekUsingNext() && + (i.exhaustedBounds == +1 || (i.data.isDataInvalidated() && i.index.isDataInvalidated())) && + err == nil { + // Already exhausted, so return nil. + return nil, base.LazyValue{} + } + + // SeekGE performs various step-instead-of-seeking optimizations: eg enabled + // by trySeekUsingNext, or by monotonically increasing bounds (i.boundsCmp). + // Care must be taken to ensure that when performing these optimizations and + // the iterator becomes exhausted, i.maybeFilteredKeys is set appropriately. + // Consider a previous SeekGE that filtered keys from k until the current + // iterator position. + // + // If the previous SeekGE exhausted the iterator while seeking within the + // two-level index, it's possible keys greater than or equal to the current + // search key were filtered through skipped index blocks. We must not reuse + // the position of the two-level index iterator without remembering the + // previous value of maybeFilteredKeys. + + // We fall into the slow path if i.index.isDataInvalidated() even if the + // top-level iterator is already positioned correctly and all other + // conditions are met. An alternative structure could reuse topLevelIndex's + // current position and reload the index block to which it points. Arguably, + // an index block load is expensive and the index block may still be earlier + // than the index block containing the sought key, resulting in a wasteful + // block load. + + var dontSeekWithinSingleLevelIter bool + if i.topLevelIndex.isDataInvalidated() || !i.topLevelIndex.valid() || i.index.isDataInvalidated() || err != nil || + (i.boundsCmp <= 0 && !flags.TrySeekUsingNext()) || i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 { + // Slow-path: need to position the topLevelIndex. + + // The previous exhausted state of singleLevelIterator is no longer + // relevant, since we may be moving to a different index block. + i.exhaustedBounds = 0 + i.maybeFilteredKeysTwoLevel = false + flags = flags.DisableTrySeekUsingNext() + var ikey *InternalKey + if ikey, _ = i.topLevelIndex.SeekGE(key, flags); ikey == nil { + i.data.invalidate() + i.index.invalidate() + return nil, base.LazyValue{} + } + + result := i.loadIndex(+1) + if result == loadBlockFailed { + i.boundsCmp = 0 + return nil, base.LazyValue{} + } + if result == loadBlockIrrelevant { + // Enforce the upper bound here since don't want to bother moving + // to the next entry in the top level index if upper bound is + // already exceeded. Note that the next entry starts with keys >= + // ikey.UserKey since even though this is the block separator, the + // same user key can span multiple index blocks. If upper is + // exclusive we use >= below, else we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + } + } + // Fall through to skipForward. + dontSeekWithinSingleLevelIter = true + // Clear boundsCmp. + // + // In the typical cases where dontSeekWithinSingleLevelIter=false, + // the singleLevelIterator.SeekGE call will clear boundsCmp. + // However, in this case where dontSeekWithinSingleLevelIter=true, + // we never seek on the single-level iterator. This call will fall + // through to skipForward, which may improperly leave boundsCmp=+1 + // unless we clear it here. + i.boundsCmp = 0 + } + } else { + // INVARIANT: err == nil. + // + // Else fast-path: There are two possible cases, from + // (i.boundsCmp > 0 || flags.TrySeekUsingNext()): + // + // 1) The bounds have moved forward (i.boundsCmp > 0) and this SeekGE is + // respecting the lower bound (guaranteed by Iterator). We know that the + // iterator must already be positioned within or just outside the previous + // bounds. Therefore, the topLevelIndex iter cannot be positioned at an + // entry ahead of the seek position (though it can be positioned behind). + // The !i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 confirms that it is + // not behind. Since it is not ahead and not behind it must be at the + // right position. + // + // 2) This SeekGE will land on a key that is greater than the key we are + // currently at (guaranteed by trySeekUsingNext), but since i.cmp(key, + // i.topLevelIndex.Key().UserKey) <= 0, we are at the correct lower level + // index block. No need to reset the state of singleLevelIterator. + // + // Note that cases 1 and 2 never overlap, and one of them must be true, + // but we have some test code (TestIterRandomizedMaybeFilteredKeys) that + // sets both to true, so we fix things here and then do an invariant + // check. + // + // This invariant checking is important enough that we do not gate it + // behind invariants.Enabled. + if i.boundsCmp > 0 { + // TODO(sumeer): fix TestIterRandomizedMaybeFilteredKeys so as to not + // need this behavior. + flags = flags.DisableTrySeekUsingNext() + } + if i.boundsCmp > 0 == flags.TrySeekUsingNext() { + panic(fmt.Sprintf("inconsistency in optimization case 1 %t and case 2 %t", + i.boundsCmp > 0, flags.TrySeekUsingNext())) + } + + if !flags.TrySeekUsingNext() { + // Case 1. Bounds have changed so the previous exhausted bounds state is + // irrelevant. + // WARNING-data-exhausted: this is safe to do only because the monotonic + // bounds optimizations only work when !data-exhausted. If they also + // worked with data-exhausted, we have made it unclear whether + // data-exhausted is actually true. See the comment at the top of the + // file. + i.exhaustedBounds = 0 + } + // Else flags.TrySeekUsingNext(). The i.exhaustedBounds is important to + // preserve for singleLevelIterator, and twoLevelIterator.skipForward. See + // bug https://github.com/cockroachdb/pebble/issues/2036. + } + + if !dontSeekWithinSingleLevelIter { + // Note that while trySeekUsingNext could be false here, singleLevelIterator + // could do its own boundsCmp-based optimization to seek using next. + if ikey, val := i.singleLevelIterator.SeekGE(key, flags); ikey != nil { + return ikey, val + } + } + return i.skipForward() +} + +// SeekPrefixGE implements internalIterator.SeekPrefixGE, as documented in the +// pebble package. Note that SeekPrefixGE only checks the upper bound. It is up +// to the caller to ensure that key is greater than or equal to the lower bound. +func (i *twoLevelIterator) SeekPrefixGE( + prefix, key []byte, flags base.SeekGEFlags, +) (*base.InternalKey, base.LazyValue) { + if i.vState != nil { + // Callers of SeekGE don't know about virtual sstable bounds, so we may + // have to internally restrict the bounds. + // + // TODO(bananabrick): We can optimize away this check for the level iter + // if necessary. + if i.cmp(key, i.lower) < 0 { + key = i.lower + } + } + + // NOTE: prefix is only used for bloom filter checking and not later work in + // this method. Hence, we can use the existing iterator position if the last + // SeekPrefixGE did not fail bloom filter matching. + + err := i.err + i.err = nil // clear cached iteration error + + // The twoLevelIterator could be already exhausted. Utilize that when + // trySeekUsingNext is true. See the comment about data-exhausted, PGDE, and + // bounds-exhausted near the top of the file. + filterUsedAndDidNotMatch := + i.reader.tableFilter != nil && i.useFilter && !i.lastBloomFilterMatched + if flags.TrySeekUsingNext() && !filterUsedAndDidNotMatch && + (i.exhaustedBounds == +1 || (i.data.isDataInvalidated() && i.index.isDataInvalidated())) && + err == nil { + // Already exhausted, so return nil. + return nil, base.LazyValue{} + } + + // Check prefix bloom filter. + if i.reader.tableFilter != nil && i.useFilter { + if !i.lastBloomFilterMatched { + // Iterator is not positioned based on last seek. + flags = flags.DisableTrySeekUsingNext() + } + i.lastBloomFilterMatched = false + var dataH bufferHandle + dataH, i.err = i.reader.readFilter(i.ctx, i.stats) + if i.err != nil { + i.data.invalidate() + return nil, base.LazyValue{} + } + mayContain := i.reader.tableFilter.mayContain(dataH.Get(), prefix) + dataH.Release() + if !mayContain { + // This invalidation may not be necessary for correctness, and may + // be a place to optimize later by reusing the already loaded + // block. It was necessary in earlier versions of the code since + // the caller was allowed to call Next when SeekPrefixGE returned + // nil. This is no longer allowed. + i.data.invalidate() + return nil, base.LazyValue{} + } + i.lastBloomFilterMatched = true + } + + // Bloom filter matches. + + // SeekPrefixGE performs various step-instead-of-seeking optimizations: eg + // enabled by trySeekUsingNext, or by monotonically increasing bounds + // (i.boundsCmp). Care must be taken to ensure that when performing these + // optimizations and the iterator becomes exhausted, + // i.maybeFilteredKeysTwoLevel is set appropriately. Consider a previous + // SeekPrefixGE that filtered keys from k until the current iterator + // position. + // + // If the previous SeekPrefixGE exhausted the iterator while seeking within + // the two-level index, it's possible keys greater than or equal to the + // current search key were filtered through skipped index blocks. We must + // not reuse the position of the two-level index iterator without + // remembering the previous value of maybeFilteredKeysTwoLevel. + + // We fall into the slow path if i.index.isDataInvalidated() even if the + // top-level iterator is already positioned correctly and all other + // conditions are met. An alternative structure could reuse topLevelIndex's + // current position and reload the index block to which it points. Arguably, + // an index block load is expensive and the index block may still be earlier + // than the index block containing the sought key, resulting in a wasteful + // block load. + + var dontSeekWithinSingleLevelIter bool + if i.topLevelIndex.isDataInvalidated() || !i.topLevelIndex.valid() || i.index.isDataInvalidated() || err != nil || + (i.boundsCmp <= 0 && !flags.TrySeekUsingNext()) || i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 { + // Slow-path: need to position the topLevelIndex. + + // The previous exhausted state of singleLevelIterator is no longer + // relevant, since we may be moving to a different index block. + i.exhaustedBounds = 0 + i.maybeFilteredKeysTwoLevel = false + flags = flags.DisableTrySeekUsingNext() + var ikey *InternalKey + if ikey, _ = i.topLevelIndex.SeekGE(key, flags); ikey == nil { + i.data.invalidate() + i.index.invalidate() + return nil, base.LazyValue{} + } + + result := i.loadIndex(+1) + if result == loadBlockFailed { + i.boundsCmp = 0 + return nil, base.LazyValue{} + } + if result == loadBlockIrrelevant { + // Enforce the upper bound here since don't want to bother moving + // to the next entry in the top level index if upper bound is + // already exceeded. Note that the next entry starts with keys >= + // ikey.UserKey since even though this is the block separator, the + // same user key can span multiple index blocks. If upper is + // exclusive we use >= below, else we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + } + } + // Fall through to skipForward. + dontSeekWithinSingleLevelIter = true + // Clear boundsCmp. + // + // In the typical cases where dontSeekWithinSingleLevelIter=false, + // the singleLevelIterator.SeekPrefixGE call will clear boundsCmp. + // However, in this case where dontSeekWithinSingleLevelIter=true, + // we never seek on the single-level iterator. This call will fall + // through to skipForward, which may improperly leave boundsCmp=+1 + // unless we clear it here. + i.boundsCmp = 0 + } + } else { + // INVARIANT: err == nil. + // + // Else fast-path: There are two possible cases, from + // (i.boundsCmp > 0 || flags.TrySeekUsingNext()): + // + // 1) The bounds have moved forward (i.boundsCmp > 0) and this + // SeekPrefixGE is respecting the lower bound (guaranteed by Iterator). We + // know that the iterator must already be positioned within or just + // outside the previous bounds. Therefore, the topLevelIndex iter cannot + // be positioned at an entry ahead of the seek position (though it can be + // positioned behind). The !i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 + // confirms that it is not behind. Since it is not ahead and not behind it + // must be at the right position. + // + // 2) This SeekPrefixGE will land on a key that is greater than the key we + // are currently at (guaranteed by trySeekUsingNext), but since i.cmp(key, + // i.topLevelIndex.Key().UserKey) <= 0, we are at the correct lower level + // index block. No need to reset the state of singleLevelIterator. + // + // Note that cases 1 and 2 never overlap, and one of them must be true. + // This invariant checking is important enough that we do not gate it + // behind invariants.Enabled. + if i.boundsCmp > 0 == flags.TrySeekUsingNext() { + panic(fmt.Sprintf("inconsistency in optimization case 1 %t and case 2 %t", + i.boundsCmp > 0, flags.TrySeekUsingNext())) + } + + if !flags.TrySeekUsingNext() { + // Case 1. Bounds have changed so the previous exhausted bounds state is + // irrelevant. + // WARNING-data-exhausted: this is safe to do only because the monotonic + // bounds optimizations only work when !data-exhausted. If they also + // worked with data-exhausted, we have made it unclear whether + // data-exhausted is actually true. See the comment at the top of the + // file. + i.exhaustedBounds = 0 + } + // Else flags.TrySeekUsingNext(). The i.exhaustedBounds is important to + // preserve for singleLevelIterator, and twoLevelIterator.skipForward. See + // bug https://github.com/cockroachdb/pebble/issues/2036. + } + + if !dontSeekWithinSingleLevelIter { + if ikey, val := i.singleLevelIterator.seekPrefixGE( + prefix, key, flags, false /* checkFilter */); ikey != nil { + return ikey, val + } + } + // NB: skipForward checks whether exhaustedBounds is already +1. + return i.skipForward() +} + +// virtualLast should only be called if i.vReader != nil and i.endKeyInclusive +// is true. +func (i *twoLevelIterator) virtualLast() (*InternalKey, base.LazyValue) { + if i.vState == nil { + panic("pebble: invalid call to virtualLast") + } + + // Seek to the first internal key. + ikey, _ := i.SeekGE(i.upper, base.SeekGEFlagsNone) + if i.endKeyInclusive { + // Let's say the virtual sstable upper bound is c#1, with the keys c#3, c#2, + // c#1, d, e, ... in the sstable. So, the last key in the virtual sstable is + // c#1. We can perform SeekGE(i.upper) and then keep nexting until we find + // the last key with userkey == i.upper. + // + // TODO(bananabrick): Think about how to improve this. If many internal keys + // with the same user key at the upper bound then this could be slow, but + // maybe the odds of having many internal keys with the same user key at the + // upper bound are low. + for ikey != nil && i.cmp(ikey.UserKey, i.upper) == 0 { + ikey, _ = i.Next() + } + return i.Prev() + } + // We seeked to the first key >= i.upper. + return i.Prev() +} + +// SeekLT implements internalIterator.SeekLT, as documented in the pebble +// package. Note that SeekLT only checks the lower bound. It is up to the +// caller to ensure that key is less than the upper bound. +func (i *twoLevelIterator) SeekLT( + key []byte, flags base.SeekLTFlags, +) (*InternalKey, base.LazyValue) { + if i.vState != nil { + // Might have to fix upper bound since virtual sstable bounds are not + // known to callers of SeekLT. + // + // TODO(bananabrick): We can optimize away this check for the level iter + // if necessary. + cmp := i.cmp(key, i.upper) + // key == i.upper is fine. We'll do the right thing and return the + // first internal key with user key < key. + if cmp > 0 { + return i.virtualLast() + } + } + + i.exhaustedBounds = 0 + i.err = nil // clear cached iteration error + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + + var result loadBlockResult + var ikey *InternalKey + // NB: Unlike SeekGE, we don't have a fast-path here since we don't know + // whether the topLevelIndex is positioned after the position that would + // be returned by doing i.topLevelIndex.SeekGE(). To know this we would + // need to know the index key preceding the current one. + // NB: If a bound-limited block property filter is configured, it's + // externally ensured that the filter is disabled (through returning + // Intersects=false irrespective of the block props provided) during seeks. + i.maybeFilteredKeysTwoLevel = false + if ikey, _ = i.topLevelIndex.SeekGE(key, base.SeekGEFlagsNone); ikey == nil { + if ikey, _ = i.topLevelIndex.Last(); ikey == nil { + i.data.invalidate() + i.index.invalidate() + return nil, base.LazyValue{} + } + + result = i.loadIndex(-1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockOK { + if ikey, val := i.singleLevelIterator.lastInternal(); ikey != nil { + return i.maybeVerifyKey(ikey, val) + } + // Fall through to skipBackward since the singleLevelIterator did + // not have any blocks that satisfy the block interval + // constraints, or the lower bound was reached. + } + // Else loadBlockIrrelevant, so fall through. + } else { + result = i.loadIndex(-1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockOK { + if ikey, val := i.singleLevelIterator.SeekLT(key, flags); ikey != nil { + return i.maybeVerifyKey(ikey, val) + } + // Fall through to skipBackward since the singleLevelIterator did + // not have any blocks that satisfy the block interval + // constraint, or the lower bound was reached. + } + // Else loadBlockIrrelevant, so fall through. + } + if result == loadBlockIrrelevant { + // Enforce the lower bound here since don't want to bother moving to + // the previous entry in the top level index if lower bound is already + // exceeded. Note that the previous entry starts with keys <= + // ikey.UserKey since even though this is the current block's + // separator, the same user key can span multiple index blocks. + if i.lower != nil && i.cmp(ikey.UserKey, i.lower) < 0 { + i.exhaustedBounds = -1 + } + } + // NB: skipBackward checks whether exhaustedBounds is already -1. + return i.skipBackward() +} + +// First implements internalIterator.First, as documented in the pebble +// package. Note that First only checks the upper bound. It is up to the caller +// to ensure that key is greater than or equal to the lower bound (e.g. via a +// call to SeekGE(lower)). +func (i *twoLevelIterator) First() (*InternalKey, base.LazyValue) { + // If the iterator was created on a virtual sstable, we will SeekGE to the + // lower bound instead of using First, because First does not respect + // bounds. + if i.vState != nil { + return i.SeekGE(i.lower, base.SeekGEFlagsNone) + } + + if i.lower != nil { + panic("twoLevelIterator.First() used despite lower bound") + } + i.exhaustedBounds = 0 + i.maybeFilteredKeysTwoLevel = false + i.err = nil // clear cached iteration error + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + + var ikey *InternalKey + if ikey, _ = i.topLevelIndex.First(); ikey == nil { + return nil, base.LazyValue{} + } + + result := i.loadIndex(+1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockOK { + if ikey, val := i.singleLevelIterator.First(); ikey != nil { + return ikey, val + } + // Else fall through to skipForward. + } else { + // result == loadBlockIrrelevant. Enforce the upper bound here since + // don't want to bother moving to the next entry in the top level + // index if upper bound is already exceeded. Note that the next entry + // starts with keys >= ikey.UserKey since even though this is the + // block separator, the same user key can span multiple index blocks. + // If upper is exclusive we use >= below, else we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + } + } + } + // NB: skipForward checks whether exhaustedBounds is already +1. + return i.skipForward() +} + +// Last implements internalIterator.Last, as documented in the pebble +// package. Note that Last only checks the lower bound. It is up to the caller +// to ensure that key is less than the upper bound (e.g. via a call to +// SeekLT(upper)) +func (i *twoLevelIterator) Last() (*InternalKey, base.LazyValue) { + if i.vState != nil { + if i.endKeyInclusive { + return i.virtualLast() + } + return i.SeekLT(i.upper, base.SeekLTFlagsNone) + } + + if i.upper != nil { + panic("twoLevelIterator.Last() used despite upper bound") + } + i.exhaustedBounds = 0 + i.maybeFilteredKeysTwoLevel = false + i.err = nil // clear cached iteration error + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + + var ikey *InternalKey + if ikey, _ = i.topLevelIndex.Last(); ikey == nil { + return nil, base.LazyValue{} + } + + result := i.loadIndex(-1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockOK { + if ikey, val := i.singleLevelIterator.Last(); ikey != nil { + return ikey, val + } + // Else fall through to skipBackward. + } else { + // result == loadBlockIrrelevant. Enforce the lower bound here + // since don't want to bother moving to the previous entry in the + // top level index if lower bound is already exceeded. Note that + // the previous entry starts with keys <= ikey.UserKey since even + // though this is the current block's separator, the same user key + // can span multiple index blocks. + if i.lower != nil && i.cmp(ikey.UserKey, i.lower) < 0 { + i.exhaustedBounds = -1 + } + } + // NB: skipBackward checks whether exhaustedBounds is already -1. + return i.skipBackward() +} + +// Next implements internalIterator.Next, as documented in the pebble +// package. +// Note: twoLevelCompactionIterator.Next mirrors the implementation of +// twoLevelIterator.Next due to performance. Keep the two in sync. +func (i *twoLevelIterator) Next() (*InternalKey, base.LazyValue) { + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + i.maybeFilteredKeysTwoLevel = false + if i.err != nil { + return nil, base.LazyValue{} + } + if key, val := i.singleLevelIterator.Next(); key != nil { + return key, val + } + return i.skipForward() +} + +// NextPrefix implements (base.InternalIterator).NextPrefix. +func (i *twoLevelIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { + if i.exhaustedBounds == +1 { + panic("Next called even though exhausted upper bound") + } + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + i.maybeFilteredKeysTwoLevel = false + if i.err != nil { + return nil, base.LazyValue{} + } + if key, val := i.singleLevelIterator.NextPrefix(succKey); key != nil { + return key, val + } + // key == nil + if i.err != nil { + return nil, base.LazyValue{} + } + + // Did not find prefix in the existing second-level index block. This is the + // slow-path where we seek the iterator. + var ikey *InternalKey + if ikey, _ = i.topLevelIndex.SeekGE(succKey, base.SeekGEFlagsNone); ikey == nil { + i.data.invalidate() + i.index.invalidate() + return nil, base.LazyValue{} + } + result := i.loadIndex(+1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockIrrelevant { + // Enforce the upper bound here since don't want to bother moving to the + // next entry in the top level index if upper bound is already exceeded. + // Note that the next entry starts with keys >= ikey.UserKey since even + // though this is the block separator, the same user key can span multiple + // index blocks. If upper is exclusive we use >= below, else we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + } + } + } else if key, val := i.singleLevelIterator.SeekGE(succKey, base.SeekGEFlagsNone); key != nil { + return i.maybeVerifyKey(key, val) + } + return i.skipForward() +} + +// Prev implements internalIterator.Prev, as documented in the pebble +// package. +func (i *twoLevelIterator) Prev() (*InternalKey, base.LazyValue) { + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + i.maybeFilteredKeysTwoLevel = false + if i.err != nil { + return nil, base.LazyValue{} + } + if key, val := i.singleLevelIterator.Prev(); key != nil { + return key, val + } + return i.skipBackward() +} + +func (i *twoLevelIterator) skipForward() (*InternalKey, base.LazyValue) { + for { + if i.err != nil || i.exhaustedBounds > 0 { + return nil, base.LazyValue{} + } + i.exhaustedBounds = 0 + var ikey *InternalKey + if ikey, _ = i.topLevelIndex.Next(); ikey == nil { + i.data.invalidate() + i.index.invalidate() + return nil, base.LazyValue{} + } + result := i.loadIndex(+1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockOK { + if ikey, val := i.singleLevelIterator.firstInternal(); ikey != nil { + return i.maybeVerifyKey(ikey, val) + } + // Next iteration will return if singleLevelIterator set + // exhaustedBounds = +1. + } else { + // result == loadBlockIrrelevant. Enforce the upper bound here + // since don't want to bother moving to the next entry in the top + // level index if upper bound is already exceeded. Note that the + // next entry starts with keys >= ikey.UserKey since even though + // this is the block separator, the same user key can span + // multiple index blocks. If upper is exclusive we use >= + // below, else we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + // Next iteration will return. + } + } + } + } +} + +func (i *twoLevelIterator) skipBackward() (*InternalKey, base.LazyValue) { + for { + if i.err != nil || i.exhaustedBounds < 0 { + return nil, base.LazyValue{} + } + i.exhaustedBounds = 0 + var ikey *InternalKey + if ikey, _ = i.topLevelIndex.Prev(); ikey == nil { + i.data.invalidate() + i.index.invalidate() + return nil, base.LazyValue{} + } + result := i.loadIndex(-1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockOK { + if ikey, val := i.singleLevelIterator.lastInternal(); ikey != nil { + return i.maybeVerifyKey(ikey, val) + } + // Next iteration will return if singleLevelIterator set + // exhaustedBounds = -1. + } else { + // result == loadBlockIrrelevant. Enforce the lower bound here + // since don't want to bother moving to the previous entry in the + // top level index if lower bound is already exceeded. Note that + // the previous entry starts with keys <= ikey.UserKey since even + // though this is the current block's separator, the same user key + // can span multiple index blocks. + if i.lower != nil && i.cmp(ikey.UserKey, i.lower) < 0 { + i.exhaustedBounds = -1 + // Next iteration will return. + } + } + } +} + +// Close implements internalIterator.Close, as documented in the pebble +// package. +func (i *twoLevelIterator) Close() error { + var err error + if i.closeHook != nil { + err = firstError(err, i.closeHook(i)) + } + err = firstError(err, i.data.Close()) + err = firstError(err, i.index.Close()) + err = firstError(err, i.topLevelIndex.Close()) + if i.dataRH != nil { + err = firstError(err, i.dataRH.Close()) + i.dataRH = nil + } + err = firstError(err, i.err) + if i.bpfs != nil { + releaseBlockPropertiesFilterer(i.bpfs) + } + if i.vbReader != nil { + i.vbReader.close() + } + if i.vbRH != nil { + err = firstError(err, i.vbRH.Close()) + i.vbRH = nil + } + *i = twoLevelIterator{ + singleLevelIterator: i.singleLevelIterator.resetForReuse(), + topLevelIndex: i.topLevelIndex.resetForReuse(), + } + twoLevelIterPool.Put(i) + return err +} + +// Note: twoLevelCompactionIterator and compactionIterator are very similar but +// were separated due to performance. +type twoLevelCompactionIterator struct { + *twoLevelIterator + bytesIterated *uint64 + prevOffset uint64 +} + +// twoLevelCompactionIterator implements the base.InternalIterator interface. +var _ base.InternalIterator = (*twoLevelCompactionIterator)(nil) + +func (i *twoLevelCompactionIterator) Close() error { + return i.twoLevelIterator.Close() +} + +func (i *twoLevelCompactionIterator) SeekGE( + key []byte, flags base.SeekGEFlags, +) (*InternalKey, base.LazyValue) { + panic("pebble: SeekGE unimplemented") +} + +func (i *twoLevelCompactionIterator) SeekPrefixGE( + prefix, key []byte, flags base.SeekGEFlags, +) (*base.InternalKey, base.LazyValue) { + panic("pebble: SeekPrefixGE unimplemented") +} + +func (i *twoLevelCompactionIterator) SeekLT( + key []byte, flags base.SeekLTFlags, +) (*InternalKey, base.LazyValue) { + panic("pebble: SeekLT unimplemented") +} + +func (i *twoLevelCompactionIterator) First() (*InternalKey, base.LazyValue) { + i.err = nil // clear cached iteration error + return i.skipForward(i.twoLevelIterator.First()) +} + +func (i *twoLevelCompactionIterator) Last() (*InternalKey, base.LazyValue) { + panic("pebble: Last unimplemented") +} + +// Note: twoLevelCompactionIterator.Next mirrors the implementation of +// twoLevelIterator.Next due to performance. Keep the two in sync. +func (i *twoLevelCompactionIterator) Next() (*InternalKey, base.LazyValue) { + if i.err != nil { + return nil, base.LazyValue{} + } + return i.skipForward(i.singleLevelIterator.Next()) +} + +func (i *twoLevelCompactionIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { + panic("pebble: NextPrefix unimplemented") +} + +func (i *twoLevelCompactionIterator) Prev() (*InternalKey, base.LazyValue) { + panic("pebble: Prev unimplemented") +} + +func (i *twoLevelCompactionIterator) String() string { + if i.vState != nil { + return i.vState.fileNum.String() + } + return i.reader.fileNum.String() +} + +func (i *twoLevelCompactionIterator) skipForward( + key *InternalKey, val base.LazyValue, +) (*InternalKey, base.LazyValue) { + if key == nil { + for { + if key, _ := i.topLevelIndex.Next(); key == nil { + break + } + result := i.loadIndex(+1) + if result != loadBlockOK { + if i.err != nil { + break + } + switch result { + case loadBlockFailed: + // We checked that i.index was at a valid entry, so + // loadBlockFailed could not have happened due to to i.index + // being exhausted, and must be due to an error. + panic("loadBlock should not have failed with no error") + case loadBlockIrrelevant: + panic("compactionIter should not be using block intervals for skipping") + default: + panic(fmt.Sprintf("unexpected case %d", result)) + } + } + // result == loadBlockOK + if key, val = i.singleLevelIterator.First(); key != nil { + break + } + } + } + + curOffset := i.recordOffset() + *i.bytesIterated += uint64(curOffset - i.prevOffset) + i.prevOffset = curOffset + + if i.vState != nil && key != nil { + cmp := i.cmp(key.UserKey, i.vState.upper.UserKey) + if cmp > 0 || (i.vState.upper.IsExclusiveSentinel() && cmp == 0) { + return nil, base.LazyValue{} + } + } + + return key, val +} diff --git a/vendor/github.com/cockroachdb/pebble/sstable/reader_virtual.go b/vendor/github.com/cockroachdb/pebble/sstable/reader_virtual.go new file mode 100644 index 0000000..b044d19 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/sstable/reader_virtual.go @@ -0,0 +1,206 @@ +// Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package sstable + +import ( + "context" + + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/keyspan" + "github.com/cockroachdb/pebble/internal/manifest" +) + +// VirtualReader wraps Reader. Its purpose is to restrict functionality of the +// Reader which should be inaccessible to virtual sstables, and enforce bounds +// invariants associated with virtual sstables. All reads on virtual sstables +// should go through a VirtualReader. +// +// INVARIANT: Any iterators created through a virtual reader will guarantee that +// they don't expose keys outside the virtual sstable bounds. +type VirtualReader struct { + vState virtualState + reader *Reader + Properties CommonProperties +} + +// Lightweight virtual sstable state which can be passed to sstable iterators. +type virtualState struct { + lower InternalKey + upper InternalKey + fileNum base.FileNum + Compare Compare + isForeign bool +} + +func ceilDiv(a, b uint64) uint64 { + return (a + b - 1) / b +} + +// MakeVirtualReader is used to contruct a reader which can read from virtual +// sstables. +func MakeVirtualReader( + reader *Reader, meta manifest.VirtualFileMeta, isForeign bool, +) VirtualReader { + if reader.fileNum != meta.FileBacking.DiskFileNum { + panic("pebble: invalid call to MakeVirtualReader") + } + + vState := virtualState{ + lower: meta.Smallest, + upper: meta.Largest, + fileNum: meta.FileNum, + Compare: reader.Compare, + isForeign: isForeign, + } + v := VirtualReader{ + vState: vState, + reader: reader, + } + + v.Properties.RawKeySize = ceilDiv(reader.Properties.RawKeySize*meta.Size, meta.FileBacking.Size) + v.Properties.RawValueSize = ceilDiv(reader.Properties.RawValueSize*meta.Size, meta.FileBacking.Size) + v.Properties.NumEntries = ceilDiv(reader.Properties.NumEntries*meta.Size, meta.FileBacking.Size) + v.Properties.NumDeletions = ceilDiv(reader.Properties.NumDeletions*meta.Size, meta.FileBacking.Size) + v.Properties.NumRangeDeletions = ceilDiv(reader.Properties.NumRangeDeletions*meta.Size, meta.FileBacking.Size) + v.Properties.NumRangeKeyDels = ceilDiv(reader.Properties.NumRangeKeyDels*meta.Size, meta.FileBacking.Size) + + // Note that we rely on NumRangeKeySets for correctness. If the sstable may + // contain range keys, then NumRangeKeySets must be > 0. ceilDiv works because + // meta.Size will not be 0 for virtual sstables. + v.Properties.NumRangeKeySets = ceilDiv(reader.Properties.NumRangeKeySets*meta.Size, meta.FileBacking.Size) + v.Properties.ValueBlocksSize = ceilDiv(reader.Properties.ValueBlocksSize*meta.Size, meta.FileBacking.Size) + v.Properties.NumSizedDeletions = ceilDiv(reader.Properties.NumSizedDeletions*meta.Size, meta.FileBacking.Size) + v.Properties.RawPointTombstoneKeySize = ceilDiv(reader.Properties.RawPointTombstoneKeySize*meta.Size, meta.FileBacking.Size) + v.Properties.RawPointTombstoneValueSize = ceilDiv(reader.Properties.RawPointTombstoneValueSize*meta.Size, meta.FileBacking.Size) + return v +} + +// NewCompactionIter is the compaction iterator function for virtual readers. +func (v *VirtualReader) NewCompactionIter( + bytesIterated *uint64, rp ReaderProvider, bufferPool *BufferPool, +) (Iterator, error) { + return v.reader.newCompactionIter(bytesIterated, rp, &v.vState, bufferPool) +} + +// NewIterWithBlockPropertyFiltersAndContextEtc wraps +// Reader.NewIterWithBlockPropertyFiltersAndContext. We assume that the passed +// in [lower, upper) bounds will have at least some overlap with the virtual +// sstable bounds. No overlap is not currently supported in the iterator. +func (v *VirtualReader) NewIterWithBlockPropertyFiltersAndContextEtc( + ctx context.Context, + lower, upper []byte, + filterer *BlockPropertiesFilterer, + hideObsoletePoints, useFilterBlock bool, + stats *base.InternalIteratorStats, + rp ReaderProvider, +) (Iterator, error) { + return v.reader.newIterWithBlockPropertyFiltersAndContext( + ctx, lower, upper, filterer, hideObsoletePoints, useFilterBlock, stats, rp, &v.vState, + ) +} + +// ValidateBlockChecksumsOnBacking will call ValidateBlockChecksumsOnBacking on the underlying reader. +// Note that block checksum validation is NOT restricted to virtual sstable bounds. +func (v *VirtualReader) ValidateBlockChecksumsOnBacking() error { + return v.reader.ValidateBlockChecksums() +} + +// NewRawRangeDelIter wraps Reader.NewRawRangeDelIter. +func (v *VirtualReader) NewRawRangeDelIter() (keyspan.FragmentIterator, error) { + iter, err := v.reader.NewRawRangeDelIter() + if err != nil { + return nil, err + } + if iter == nil { + return nil, nil + } + + // Truncation of spans isn't allowed at a user key that also contains points + // in the same virtual sstable, as it would lead to covered points getting + // uncovered. Set panicOnUpperTruncate to true if the file's upper bound + // is not an exclusive sentinel. + // + // As an example, if an sstable contains a rangedel a-c and point keys at + // a.SET.2 and b.SET.3, the file bounds [a#2,SET-b#RANGEDELSENTINEL] are + // allowed (as they exclude b.SET.3), or [a#2,SET-c#RANGEDELSENTINEL] (as it + // includes both point keys), but not [a#2,SET-b#3,SET] (as it would truncate + // the rangedel at b and lead to the point being uncovered). + return keyspan.Truncate( + v.reader.Compare, iter, v.vState.lower.UserKey, v.vState.upper.UserKey, + &v.vState.lower, &v.vState.upper, !v.vState.upper.IsExclusiveSentinel(), /* panicOnUpperTruncate */ + ), nil +} + +// NewRawRangeKeyIter wraps Reader.NewRawRangeKeyIter. +func (v *VirtualReader) NewRawRangeKeyIter() (keyspan.FragmentIterator, error) { + iter, err := v.reader.NewRawRangeKeyIter() + if err != nil { + return nil, err + } + if iter == nil { + return nil, nil + } + + // Truncation of spans isn't allowed at a user key that also contains points + // in the same virtual sstable, as it would lead to covered points getting + // uncovered. Set panicOnUpperTruncate to true if the file's upper bound + // is not an exclusive sentinel. + // + // As an example, if an sstable contains a range key a-c and point keys at + // a.SET.2 and b.SET.3, the file bounds [a#2,SET-b#RANGEKEYSENTINEL] are + // allowed (as they exclude b.SET.3), or [a#2,SET-c#RANGEKEYSENTINEL] (as it + // includes both point keys), but not [a#2,SET-b#3,SET] (as it would truncate + // the range key at b and lead to the point being uncovered). + return keyspan.Truncate( + v.reader.Compare, iter, v.vState.lower.UserKey, v.vState.upper.UserKey, + &v.vState.lower, &v.vState.upper, !v.vState.upper.IsExclusiveSentinel(), /* panicOnUpperTruncate */ + ), nil +} + +// Constrain bounds will narrow the start, end bounds if they do not fit within +// the virtual sstable. The function will return if the new end key is +// inclusive. +func (v *virtualState) constrainBounds( + start, end []byte, endInclusive bool, +) (lastKeyInclusive bool, first []byte, last []byte) { + first = start + if start == nil || v.Compare(start, v.lower.UserKey) < 0 { + first = v.lower.UserKey + } + + // Note that we assume that start, end has some overlap with the virtual + // sstable bounds. + last = v.upper.UserKey + lastKeyInclusive = !v.upper.IsExclusiveSentinel() + if end != nil { + cmp := v.Compare(end, v.upper.UserKey) + switch { + case cmp == 0: + lastKeyInclusive = !v.upper.IsExclusiveSentinel() && endInclusive + last = v.upper.UserKey + case cmp > 0: + lastKeyInclusive = !v.upper.IsExclusiveSentinel() + last = v.upper.UserKey + default: + lastKeyInclusive = endInclusive + last = end + } + } + // TODO(bananabrick): What if someone passes in bounds completely outside of + // virtual sstable bounds? + return lastKeyInclusive, first, last +} + +// EstimateDiskUsage just calls VirtualReader.reader.EstimateDiskUsage after +// enforcing the virtual sstable bounds. +func (v *VirtualReader) EstimateDiskUsage(start, end []byte) (uint64, error) { + _, f, l := v.vState.constrainBounds(start, end, true /* endInclusive */) + return v.reader.EstimateDiskUsage(f, l) +} + +// CommonProperties implements the CommonReader interface. +func (v *VirtualReader) CommonProperties() *CommonProperties { + return &v.Properties +} diff --git a/vendor/github.com/cockroachdb/pebble/sstable/suffix_rewriter.go b/vendor/github.com/cockroachdb/pebble/sstable/suffix_rewriter.go index aee7024..9672ded 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/suffix_rewriter.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/suffix_rewriter.go @@ -2,6 +2,7 @@ package sstable import ( "bytes" + "context" "math" "sync" @@ -42,6 +43,9 @@ func RewriteKeySuffixes( // modification as they are not affected by the suffix, while block and table // properties are only minimally recomputed. // +// TODO(sumeer): document limitations, if any, due to this limited +// re-computation of properties (is there any loss of fidelity?). +// // Any block and table property collectors configured in the WriterOptions must // implement SuffixReplaceableTableCollector/SuffixReplaceableBlockCollector. // @@ -49,6 +53,26 @@ func RewriteKeySuffixes( // same TableFormat as the input, which is returned in case the caller wants // to do some error checking. Suffix rewriting is meant to be efficient, and // allowing changes in the TableFormat detracts from that efficiency. +// +// Any obsolete bits that key-value pairs may be annotated with are ignored +// and lost during the rewrite. Additionally, the output sstable has the +// pebble.obsolete.is_strict property set to false. These limitations could be +// removed if needed. The current use case for +// RewriteKeySuffixesAndReturnFormat in CockroachDB is for MVCC-compliant file +// ingestion, where these files do not contain RANGEDELs and have one +// key-value pair per userkey -- so they trivially satisfy the strict +// criteria, and we don't need the obsolete bit as a performance optimization. +// For disaggregated storage, strict obsolete sstables are needed for L5 and +// L6, but at the time of writing, we expect such MVCC-compliant file +// ingestion to only ingest into levels L4 and higher. If this changes, we can +// do one of two things to get rid of this limitation: +// - Validate that there are no duplicate userkeys and no RANGEDELs/MERGEs +// in the sstable to be rewritten. Validating no duplicate userkeys is +// non-trivial when rewriting blocks in parallel, so we could encode the +// pre-existing condition in the (existing) SnapshotPinnedKeys property -- +// we need to update the external sst writer to calculate and encode this +// property. +// - Preserve the obsolete bit (with changes to the blockIter). func RewriteKeySuffixesAndReturnFormat( sst []byte, rOpts ReaderOptions, @@ -183,7 +207,7 @@ func rewriteBlocks( if err != nil { return err } - if err := iter.init(r.Compare, inputBlock, r.Properties.GlobalSeqNum); err != nil { + if err := iter.init(r.Compare, inputBlock, r.Properties.GlobalSeqNum, false); err != nil { return err } @@ -217,12 +241,12 @@ func rewriteBlocks( copy(scratch.UserKey, key.UserKey[:si]) copy(scratch.UserKey[si:], to) - // NB: for TableFormatPebblev3, since + // NB: for TableFormatPebblev3 and higher, since // !iter.lazyValueHandling.hasValuePrefix, it will return the raw value // in the block, which includes the 1-byte prefix. This is fine since bw // also does not know about the prefix and will preserve it in bw.add. v := val.InPlaceValue() - if invariants.Enabled && r.tableFormat == TableFormatPebblev3 && + if invariants.Enabled && r.tableFormat >= TableFormatPebblev3 && key.Kind() == InternalKeyKindSet { if len(v) < 1 { return errors.Errorf("value has no prefix") @@ -432,8 +456,13 @@ func (c copyFilterWriter) policyName() string { return c.origPolicyName } // RewriteKeySuffixesViaWriter is similar to RewriteKeySuffixes but uses just a // single loop over the Reader that writes each key to the Writer with the new // suffix. The is significantly slower than the parallelized rewriter, and does -// more work to rederive filters, props, etc, however re-doing that work makes -// it less restrictive -- props no longer need to +// more work to rederive filters, props, etc. +// +// Any obsolete bits that key-value pairs may be annotated with are ignored +// and lost during the rewrite. Some of the obsolete bits may be recreated -- +// specifically when there are multiple keys with the same user key. +// Additionally, the output sstable has the pebble.obsolete.is_strict property +// set to false. See the longer comment at RewriteKeySuffixesAndReturnFormat. func RewriteKeySuffixesViaWriter( r *Reader, out objstorage.Writable, o WriterOptions, from, to []byte, ) (*WriterMetadata, error) { @@ -441,6 +470,7 @@ func RewriteKeySuffixesViaWriter( return nil, errors.New("a valid splitter is required to rewrite suffixes") } + o.IsStrictObsolete = false w := NewWriter(out, o) defer func() { if w != nil { @@ -471,7 +501,7 @@ func RewriteKeySuffixesViaWriter( if err != nil { return nil, err } - if w.addPoint(scratch, val); err != nil { + if w.addPoint(scratch, val, false); err != nil { return nil, err } k, v = i.Next() @@ -514,7 +544,7 @@ func readBlockBuf(r *Reader, bh BlockHandle, buf []byte) ([]byte, []byte, error) return res, buf, err } -// memReader is a thin wrapper around a []byte such that it can be passed to an +// memReader is a thin wrapper around a []byte such that it can be passed to // sstable.Reader. It supports concurrent use, and does so without locking in // contrast to the heavier read/write vfs.MemFile. type memReader struct { @@ -534,22 +564,26 @@ func newMemReader(b []byte) *memReader { return r } -// ReadAt implements io.ReaderAt. -func (m *memReader) ReadAt(p []byte, off int64) (n int, err error) { - return m.r.ReadAt(p, off) +// ReadAt is part of objstorage.Readable. +func (m *memReader) ReadAt(_ context.Context, p []byte, off int64) error { + n, err := m.r.ReadAt(p, off) + if invariants.Enabled && err == nil && n != len(p) { + panic("short read") + } + return err } -// Close implements io.Closer. +// Close is part of objstorage.Readable. func (*memReader) Close() error { return nil } -// Stat implements objstorage.Readable. +// Stat is part of objstorage.Readable. func (m *memReader) Size() int64 { return int64(len(m.b)) } -// NewReadHandle implements objstorage.Readable. -func (m *memReader) NewReadHandle() objstorage.ReadHandle { +// NewReadHandle is part of objstorage.Readable. +func (m *memReader) NewReadHandle(_ context.Context) objstorage.ReadHandle { return &m.rh } diff --git a/vendor/github.com/cockroachdb/pebble/sstable/table.go b/vendor/github.com/cockroachdb/pebble/sstable/table.go index 3d96684..040efaf 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/table.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/table.go @@ -68,8 +68,8 @@ package sstable // import "github.com/cockroachdb/pebble/sstable" import ( + "context" "encoding/binary" - "io" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" @@ -160,8 +160,27 @@ The metaindex block also contains block handles as values, with keys being the names of the meta blocks. For a description of value blocks and the meta value index block, see -value_block.go +value_block.go. +Data blocks have some additional features: +- For TableFormatPebblev3 onwards: + - For SETs, the value has a 1 byte value prefix, which indicates whether the + value is inline, or in a separate value block, and indicates whether the + prefix of the userkey (as defined by split) has changed or not. See + value_block.go for details. + - The most significant bit of the restart points is used to indicate whether + userkey prefix has changed since the last restart point. See the detailed + comment in blockWriter. + - The maximum length of the "shared prefix" when encoding the key, is the + length of the prefix of the userkey (as defined by split) of the previous + key. + +- For TableFormatPebblev4 onwards: + - The key kinds may be altered to set the + InternalKeyKindSSTableInternalObsoleteBit if the key-value pair is obsolete + in the context of that sstable (for a reader that reads at a higher seqnum + than the highest seqnum in the sstable). For details, see the comment in + format.go. */ const ( @@ -316,12 +335,11 @@ func readFooter(f objstorage.Readable) (footer, error) { off := size - maxFooterLen if off < 0 { off = 0 + buf = buf[:size] } - n, err := f.ReadAt(buf, off) - if err != nil && err != io.EOF { + if err := f.ReadAt(context.TODO(), buf, off); err != nil { return footer, errors.Wrap(err, "pebble/table: invalid table (could not read footer)") } - buf = buf[:n] switch magic := buf[len(buf)-len(rocksDBMagic):]; string(magic) { case levelDBMagic: @@ -429,7 +447,7 @@ func supportsTwoLevelIndex(format TableFormat) bool { switch format { case TableFormatLevelDB: return false - case TableFormatRocksDBv2, TableFormatPebblev1, TableFormatPebblev2, TableFormatPebblev3: + case TableFormatRocksDBv2, TableFormatPebblev1, TableFormatPebblev2, TableFormatPebblev3, TableFormatPebblev4: return true default: panic("sstable: unspecified table format version") diff --git a/vendor/github.com/cockroachdb/pebble/sstable/value_block.go b/vendor/github.com/cockroachdb/pebble/sstable/value_block.go index cc033df..447348b 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/value_block.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/value_block.go @@ -13,17 +13,83 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" - "github.com/cockroachdb/pebble/internal/cache" "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing" "golang.org/x/exp/rand" ) // Value blocks are supported in TableFormatPebblev3. // +// 1. Motivation and overview +// // Value blocks are a mechanism designed for sstables storing MVCC data, where // there can be many versions of a key that need to be kept, but only the -// latest value is typically read. See the documentation for Comparer.Split -// regarding MVCC keys. +// latest value is typically read (see the documentation for Comparer.Split +// regarding MVCC keys). The goal is faster reads. Unlike Pebble versions, +// which can be eagerly thrown away (except when there are snapshots), MVCC +// versions are long-lived (e.g. default CockroachDB garbage collection +// threshold for older versions is 24 hours) and can significantly slow down +// reads. We have seen CockroachDB production workloads with very slow reads +// due to: +// - 100s of versions for each key in a table. +// +// - Tables with mostly MVCC garbage consisting of 2 versions per key -- a +// real key-value pair, followed by a key-value pair whose value (usually +// with zero byte length) indicates it is an MVCC tombstone. +// +// The value blocks mechanism attempts to improve read throughput in these +// cases when the key size is smaller than the value sizes of older versions. +// This is done by moving the value of an older version to a value block in a +// different part of the sstable. This improves spatial locality of the data +// being read by the workload, which increases caching effectiveness. +// +// Additionally, even when the key size is not smaller than the value of older +// versions (e.g. secondary indexes in CockroachDB), TableFormatPebblev3 +// stores the result of key comparisons done at write time inside the sstable, +// which makes stepping from one key prefix to the next prefix (i.e., skipping +// over older versions of a MVCC key) more efficient by avoiding key +// comparisons and key decoding. See the results in +// https://github.com/cockroachdb/pebble/pull/2149 and more details in the +// comment inside BenchmarkIteratorScanNextPrefix. These improvements are also +// visible in end-to-end CockroachDB tests, as outlined in +// https://github.com/cockroachdb/cockroach/pull/96652. +// +// In TableFormatPebblev3, each SET has a one byte value prefix that tells us +// whether the value is in-place or in a value block. This 1 byte prefix +// encodes additional information: +// +// - ShortAttribute: This is an attribute of the value. Currently, CockroachDB +// uses it to represent whether the value is a tombstone or not. This avoids +// the need to fetch a value from the value block if the caller only wants +// to figure out whether it is an MVCC tombstone. The length of the value is +// another attribute that the caller can be interested in, and it is also +// accessible without reading the value in the value block (see the value +// handle in the details section). +// +// - SET-same-prefix: this enables the aforementioned optimization when +// stepping from one key prefix to the next key prefix. +// +// We further optimize this iteration over prefixes by using the restart +// points in a block to encode whether the SET at a restart point has the same +// prefix since the last restart point. This allows us to skip over restart +// points within the same block. See the comment in blockWriter, and how both +// SET-same-prefix and the restart point information is used in +// blockIter.nextPrefixV3. +// +// This flexibility of values that are in-place or in value blocks requires +// flexibility in the iterator interface. The InternalIterator interface +// returns a LazyValue instead of a byte slice. Additionally, pebble.Iterator +// allows the caller to ask for a LazyValue. See lazy_value.go for details, +// including the memory lifetime management. +// +// For historical discussions about this feature, see the issue +// https://github.com/cockroachdb/pebble/issues/1170 and the prototype in +// https://github.com/cockroachdb/pebble/pull/1443. +// +// The code in this file mainly covers value block and related encodings. We +// discuss these in the next section. +// +// 2. Details // // Note that the notion of the latest value is local to the sstable. It is // possible that that latest value has been deleted by a sstable in a higher @@ -34,7 +100,7 @@ import ( // read them. The code in this file is agnostic to the policy regarding what // should be stored in value blocks -- it allows even the latest MVCC version // to be stored in a value block. The policy decision in made in the -// sstable.Writer. See Writer.makeAddPointDecision. +// sstable.Writer. See Writer.makeAddPointDecisionV3. // // Data blocks contain two kinds of SET keys: those with in-place values and // those with a value handle. To distinguish these two cases we use a single @@ -661,7 +727,7 @@ func (ukb *UserKeyPrefixBound) IsEmpty() bool { type blockProviderWhenOpen interface { readBlockForVBR( ctx context.Context, h BlockHandle, stats *base.InternalIteratorStats, - ) (cache.Handle, error) + ) (bufferHandle, error) } type blockProviderWhenClosed struct { @@ -682,8 +748,12 @@ func (bpwc *blockProviderWhenClosed) close() { func (bpwc blockProviderWhenClosed) readBlockForVBR( ctx context.Context, h BlockHandle, stats *base.InternalIteratorStats, -) (cache.Handle, error) { - return bpwc.r.readBlock(ctx, h, nil, nil, stats) +) (bufferHandle, error) { + ctx = objiotracing.WithBlockType(ctx, objiotracing.ValueBlock) + // TODO(jackson,sumeer): Consider whether to use a buffer pool in this case. + // The bpwc is not allowed to outlive the iterator tree, so it cannot + // outlive the buffer pool. + return bpwc.r.readBlock(ctx, h, nil, nil, stats, nil /* buffer pool */) } // ReaderProvider supports the implementation of blockProviderWhenClosed. @@ -722,16 +792,16 @@ type valueBlockReader struct { // The value blocks index is lazily retrieved the first time the reader // needs to read a value that resides in a value block. vbiBlock []byte - vbiCache cache.Handle + vbiCache bufferHandle // When sequentially iterating through all key-value pairs, the cost of // repeatedly getting a block that is already in the cache and releasing the - // cache.Handle can be ~40% of the cpu overhead. So the reader remembers the + // bufferHandle can be ~40% of the cpu overhead. So the reader remembers the // last value block it retrieved, in case there is locality of access, and // this value block can be used for the next value retrieval. valueBlockNum uint32 valueBlock []byte valueBlockPtr unsafe.Pointer - valueCache cache.Handle + valueCache bufferHandle lazyFetcher base.LazyFetcher closed bool bufToMangle []byte @@ -765,12 +835,12 @@ func (r *valueBlockReader) close() { // we were to reopen this valueBlockReader and retrieve the same // Handle.value from the cache, we don't want to accidentally unref it when // attempting to unref the old handle. - r.vbiCache = cache.Handle{} + r.vbiCache = bufferHandle{} r.valueBlock = nil r.valueBlockPtr = nil r.valueCache.Release() // See comment above. - r.valueCache = cache.Handle{} + r.valueCache = bufferHandle{} r.closed = true // rp, vbih, stats remain valid, so that LazyFetcher.ValueFetcher can be // implemented. diff --git a/vendor/github.com/cockroachdb/pebble/sstable/writer.go b/vendor/github.com/cockroachdb/pebble/sstable/writer.go index 3f3827a..883ee7c 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/writer.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/writer.go @@ -119,7 +119,7 @@ type Writer struct { // the cache, providing a defense in depth against bugs which cause cache // collisions. cacheID uint64 - fileNum base.FileNum + fileNum base.DiskFileNum // The following fields are copied from Options. blockSize int blockSizeThreshold int @@ -132,6 +132,8 @@ type Writer struct { separator Separator successor Successor tableFormat TableFormat + isStrictObsolete bool + writingToLowestLevel bool cache *cache.Cache restartInterval int checksumType ChecksumType @@ -166,6 +168,7 @@ type Writer struct { props Properties propCollectors []TablePropertyCollector blockPropCollectors []BlockPropertyCollector + obsoleteCollector obsoleteKeyBlockPropertyCollector blockPropsEncoder blockPropertiesEncoder // filter accumulates the filter block. If populated, the filter ingests // either the output of w.split (i.e. a prefix extractor) if w.split is not @@ -215,6 +218,8 @@ type pointKeyInfo struct { // prefixLen uses w.split, if not nil. Only computed when w.valueBlockWriter // is not nil. prefixLen int + // True iff the point was marked obsolete. + isObsolete bool } type coordinationState struct { @@ -673,7 +678,12 @@ func (w *Writer) Set(key, value []byte) error { if w.err != nil { return w.err } - return w.addPoint(base.MakeInternalKey(key, 0, InternalKeyKindSet), value) + if w.isStrictObsolete { + return errors.Errorf("use AddWithForceObsolete") + } + // forceObsolete is false based on the assumption that no RANGEDELs in the + // sstable delete the added points. + return w.addPoint(base.MakeInternalKey(key, 0, InternalKeyKindSet), value, false) } // Delete deletes the value for the given key. The sequence number is set to @@ -685,7 +695,12 @@ func (w *Writer) Delete(key []byte) error { if w.err != nil { return w.err } - return w.addPoint(base.MakeInternalKey(key, 0, InternalKeyKindDelete), nil) + if w.isStrictObsolete { + return errors.Errorf("use AddWithForceObsolete") + } + // forceObsolete is false based on the assumption that no RANGEDELs in the + // sstable delete the added points. + return w.addPoint(base.MakeInternalKey(key, 0, InternalKeyKindDelete), nil, false) } // DeleteRange deletes all of the keys (and values) in the range [start,end) @@ -711,7 +726,13 @@ func (w *Writer) Merge(key, value []byte) error { if w.err != nil { return w.err } - return w.addPoint(base.MakeInternalKey(key, 0, InternalKeyKindMerge), value) + if w.isStrictObsolete { + return errors.Errorf("use AddWithForceObsolete") + } + // forceObsolete is false based on the assumption that no RANGEDELs in the + // sstable that delete the added points. If the user configured this writer + // to be strict-obsolete, addPoint will reject the addition of this MERGE. + return w.addPoint(base.MakeInternalKey(key, 0, InternalKeyKindMerge), value, false) } // Add adds a key/value pair to the table being written. For a given Writer, @@ -721,6 +742,24 @@ func (w *Writer) Merge(key, value []byte) error { // point entries. Additionally, range deletion tombstones must be fragmented // (i.e. by keyspan.Fragmenter). func (w *Writer) Add(key InternalKey, value []byte) error { + if w.isStrictObsolete { + return errors.Errorf("use AddWithForceObsolete") + } + return w.AddWithForceObsolete(key, value, false) +} + +// AddWithForceObsolete must be used when writing a strict-obsolete sstable. +// +// forceObsolete indicates whether the caller has determined that this key is +// obsolete even though it may be the latest point key for this userkey. This +// should be set to true for keys obsoleted by RANGEDELs, and is required for +// strict-obsolete sstables. +// +// Note that there are two properties, S1 and S2 (see comment in format.go) +// that strict-obsolete ssts must satisfy. S2, due to RANGEDELs, is solely the +// responsibility of the caller. S1 is solely the responsibility of the +// callee. +func (w *Writer) AddWithForceObsolete(key InternalKey, value []byte, forceObsolete bool) error { if w.err != nil { return w.err } @@ -735,7 +774,7 @@ func (w *Writer) Add(key InternalKey, value []byte) error { "pebble: range keys must be added via one of the RangeKey* functions") return w.err } - return w.addPoint(key, value) + return w.addPoint(key, value, forceObsolete) } func (w *Writer) makeAddPointDecisionV2(key InternalKey) error { @@ -745,12 +784,7 @@ func (w *Writer) makeAddPointDecisionV2(key InternalKey) error { return nil } if !w.disableKeyOrderChecks { - prevPointUserKey := w.dataBlockBuf.dataBlock.curKey - n := len(prevPointUserKey) - base.InternalTrailerLen - if n < 0 { - panic("corrupt key in blockWriter buffer") - } - prevPointUserKey = prevPointUserKey[:n:n] + prevPointUserKey := w.dataBlockBuf.dataBlock.getCurUserKey() cmpUser := w.compare(prevPointUserKey, key.UserKey) if cmpUser > 0 || (cmpUser == 0 && prevTrailer <= key.Trailer) { return errors.Errorf( @@ -762,9 +796,17 @@ func (w *Writer) makeAddPointDecisionV2(key InternalKey) error { return nil } +// REQUIRES: at least one point has been written to the Writer. +func (w *Writer) getLastPointUserKey() []byte { + if w.dataBlockBuf.dataBlock.nEntries == 0 { + panic(errors.AssertionFailedf("no point keys added to writer")) + } + return w.dataBlockBuf.dataBlock.getCurUserKey() +} + func (w *Writer) makeAddPointDecisionV3( key InternalKey, valueLen int, -) (setHasSamePrefix bool, writeToValueBlock bool, err error) { +) (setHasSamePrefix bool, writeToValueBlock bool, isObsolete bool, err error) { prevPointKeyInfo := w.lastPointKeyInfo w.lastPointKeyInfo.userKeyLen = len(key.UserKey) w.lastPointKeyInfo.prefixLen = w.lastPointKeyInfo.userKeyLen @@ -772,18 +814,16 @@ func (w *Writer) makeAddPointDecisionV3( w.lastPointKeyInfo.prefixLen = w.split(key.UserKey) } w.lastPointKeyInfo.trailer = key.Trailer - if w.dataBlockBuf.dataBlock.nEntries == 0 { - return false, false, nil + w.lastPointKeyInfo.isObsolete = false + if !w.meta.HasPointKeys { + return false, false, false, nil } - prevPointUserKey := w.dataBlockBuf.dataBlock.curKey - n := len(prevPointUserKey) - base.InternalTrailerLen - if n < 0 { - panic("corrupt key in blockWriter buffer") - } - prevPointUserKey = prevPointUserKey[:n:n] + keyKind := base.TrailerKind(key.Trailer) + prevPointUserKey := w.getLastPointUserKey() prevPointKey := InternalKey{UserKey: prevPointUserKey, Trailer: prevPointKeyInfo.trailer} - considerWriteToValueBlock := base.TrailerKind(prevPointKeyInfo.trailer) == InternalKeyKindSet && - base.TrailerKind(key.Trailer) == InternalKeyKindSet + prevKeyKind := base.TrailerKind(prevPointKeyInfo.trailer) + considerWriteToValueBlock := prevKeyKind == InternalKeyKindSet && + keyKind == InternalKeyKindSet if considerWriteToValueBlock && !w.requiredInPlaceValueBound.IsEmpty() { keyPrefix := key.UserKey[:w.lastPointKeyInfo.prefixLen] cmpUpper := w.compare( @@ -812,14 +852,63 @@ func (w *Writer) makeAddPointDecisionV3( } else { cmpUser = w.compare(prevPointUserKey, key.UserKey) } + // Ensure that no one adds a point key kind without considering the obsolete + // handling for that kind. + switch keyKind { + case InternalKeyKindSet, InternalKeyKindSetWithDelete, InternalKeyKindMerge, + InternalKeyKindDelete, InternalKeyKindSingleDelete, InternalKeyKindDeleteSized: + default: + panic(errors.AssertionFailedf("unexpected key kind %s", keyKind.String())) + } + // If same user key, then the current key is obsolete if any of the + // following is true: + // C1 The prev key was obsolete. + // C2 The prev key was not a MERGE. When the previous key is a MERGE we must + // preserve SET* and MERGE since their values will be merged into the + // previous key. We also must preserve DEL* since there may be an older + // SET*/MERGE in a lower level that must not be merged with the MERGE -- + // if we omit the DEL* that lower SET*/MERGE will become visible. + // + // Regardless of whether it is the same user key or not + // C3 The current key is some kind of point delete, and we are writing to + // the lowest level, then it is also obsolete. The correctness of this + // relies on the same user key not spanning multiple sstables in a level. + // + // C1 ensures that for a user key there is at most one transition from + // !obsolete to obsolete. Consider a user key k, for which the first n keys + // are not obsolete. We consider the various value of n: + // + // n = 0: This happens due to forceObsolete being set by the caller, or due + // to C3. forceObsolete must only be set due a RANGEDEL, and that RANGEDEL + // must also delete all the lower seqnums for the same user key. C3 triggers + // due to a point delete and that deletes all the lower seqnums for the same + // user key. + // + // n = 1: This is the common case. It happens when the first key is not a + // MERGE, or the current key is some kind of point delete. + // + // n > 1: This is due to a sequence of MERGE keys, potentially followed by a + // single non-MERGE key. + isObsoleteC1AndC2 := cmpUser == 0 && + (prevPointKeyInfo.isObsolete || prevKeyKind != InternalKeyKindMerge) + isObsoleteC3 := w.writingToLowestLevel && + (keyKind == InternalKeyKindDelete || keyKind == InternalKeyKindSingleDelete || + keyKind == InternalKeyKindDeleteSized) + isObsolete = isObsoleteC1AndC2 || isObsoleteC3 + // TODO(sumeer): storing isObsolete SET and SETWITHDEL in value blocks is + // possible, but requires some care in documenting and checking invariants. + // There is code that assumes nothing in value blocks because of single MVCC + // version (those should be ok). We have to ensure setHasSamePrefix is + // correctly initialized here etc. + if !w.disableKeyOrderChecks && (cmpUser > 0 || (cmpUser == 0 && prevPointKeyInfo.trailer <= key.Trailer)) { - return false, false, errors.Errorf( + return false, false, false, errors.Errorf( "pebble: keys must be added in strictly increasing order: %s, %s", prevPointKey.Pretty(w.formatKey), key.Pretty(w.formatKey)) } if !considerWriteToValueBlock { - return false, false, nil + return false, false, isObsolete, nil } // NB: it is possible that cmpUser == 0, i.e., these two SETs have identical // user keys (because of an open snapshot). This should be the rare case. @@ -832,19 +921,24 @@ func (w *Writer) makeAddPointDecisionV3( if considerWriteToValueBlock && valueLen <= tinyValueThreshold { considerWriteToValueBlock = false } - return setHasSamePrefix, considerWriteToValueBlock, nil + return setHasSamePrefix, considerWriteToValueBlock, isObsolete, nil } -func (w *Writer) addPoint(key InternalKey, value []byte) error { +func (w *Writer) addPoint(key InternalKey, value []byte, forceObsolete bool) error { + if w.isStrictObsolete && key.Kind() == InternalKeyKindMerge { + return errors.Errorf("MERGE not supported in a strict-obsolete sstable") + } var err error var setHasSameKeyPrefix, writeToValueBlock, addPrefixToValueStoredWithKey bool + var isObsolete bool maxSharedKeyLen := len(key.UserKey) if w.valueBlockWriter != nil { // maxSharedKeyLen is limited to the prefix of the preceding key. If the // preceding key was in a different block, then the blockWriter will // ignore this maxSharedKeyLen. maxSharedKeyLen = w.lastPointKeyInfo.prefixLen - setHasSameKeyPrefix, writeToValueBlock, err = w.makeAddPointDecisionV3(key, len(value)) + setHasSameKeyPrefix, writeToValueBlock, isObsolete, err = + w.makeAddPointDecisionV3(key, len(value)) addPrefixToValueStoredWithKey = base.TrailerKind(key.Trailer) == InternalKeyKindSet } else { err = w.makeAddPointDecisionV2(key) @@ -852,6 +946,8 @@ func (w *Writer) addPoint(key InternalKey, value []byte) error { if err != nil { return err } + isObsolete = w.tableFormat >= TableFormatPebblev4 && (isObsolete || forceObsolete) + w.lastPointKeyInfo.isObsolete = isObsolete var valueStoredWithKey []byte var prefix valuePrefix var valueStoredWithKeyLen int @@ -907,16 +1003,19 @@ func (w *Writer) addPoint(key InternalKey, value []byte) error { return err } } + if w.tableFormat >= TableFormatPebblev4 { + w.obsoleteCollector.AddPoint(isObsolete) + } w.maybeAddToFilter(key.UserKey) w.dataBlockBuf.dataBlock.addWithOptionalValuePrefix( - key, valueStoredWithKey, maxSharedKeyLen, addPrefixToValueStoredWithKey, prefix, + key, isObsolete, valueStoredWithKey, maxSharedKeyLen, addPrefixToValueStoredWithKey, prefix, setHasSameKeyPrefix) w.meta.updateSeqNum(key.SeqNum()) if !w.meta.HasPointKeys { - k := base.DecodeInternalKey(w.dataBlockBuf.dataBlock.curKey) + k := w.dataBlockBuf.dataBlock.getCurKey() // NB: We need to ensure that SmallestPoint.UserKey is set, so we create // an InternalKey which is semantically identical to the key, but won't // have a nil UserKey. We do this, because key.UserKey could be nil, and @@ -932,6 +1031,22 @@ func (w *Writer) addPoint(key InternalKey, value []byte) error { switch key.Kind() { case InternalKeyKindDelete, InternalKeyKindSingleDelete: w.props.NumDeletions++ + w.props.RawPointTombstoneKeySize += uint64(len(key.UserKey)) + case InternalKeyKindDeleteSized: + var size uint64 + if len(value) > 0 { + var n int + size, n = binary.Uvarint(value) + if n <= 0 { + w.err = errors.Newf("%s key's value (%x) does not parse as uvarint", + errors.Safe(key.Kind().String()), value) + return w.err + } + } + w.props.NumDeletions++ + w.props.NumSizedDeletions++ + w.props.RawPointTombstoneKeySize += uint64(len(key.UserKey)) + w.props.RawPointTombstoneValueSize += size case InternalKeyKindMerge: w.props.NumMergeOperands++ } @@ -952,7 +1067,7 @@ func (w *Writer) addTombstone(key InternalKey, value []byte) error { if !w.disableKeyOrderChecks && !w.rangeDelV1Format && w.rangeDelBlock.nEntries > 0 { // Check that tombstones are being added in fragmented order. If the two // tombstones overlap, their start and end keys must be identical. - prevKey := base.DecodeInternalKey(w.rangeDelBlock.curKey) + prevKey := w.rangeDelBlock.getCurKey() switch c := w.compare(prevKey.UserKey, key.UserKey); { case c > 0: w.err = errors.Errorf("pebble: keys must be added in order: %s, %s", @@ -1148,7 +1263,7 @@ func (w *Writer) encodeRangeKeySpan(span keyspan.Span) { func (w *Writer) addRangeKey(key InternalKey, value []byte) error { if !w.disableKeyOrderChecks && w.rangeKeyBlock.nEntries > 0 { - prevStartKey := base.DecodeInternalKey(w.rangeKeyBlock.curKey) + prevStartKey := w.rangeKeyBlock.getCurKey() prevEndKey, _, ok := rangekey.DecodeEndKey(prevStartKey.Kind(), w.rangeKeyBlock.curValue) if !ok { // We panic here as we should have previously decoded and validated this @@ -1294,7 +1409,7 @@ func (w *Writer) flush(key InternalKey) error { // dataBlockBuf is added back to a sync.Pool. In this particular case, the // byte slice which supports "sep" will eventually be copied when "sep" is // added to the index block. - prevKey := base.DecodeInternalKey(w.dataBlockBuf.dataBlock.curKey) + prevKey := w.dataBlockBuf.dataBlock.getCurKey() sep := w.indexEntrySep(prevKey, key, w.dataBlockBuf) // We determine that we should flush an index block from the Writer client // goroutine, but we actually finish the index block from the writeQueue. @@ -1584,7 +1699,7 @@ func (w *Writer) finishIndexBlock(indexBuf *indexBlockBuf, props []byte) error { nEntries: indexBuf.block.nEntries, properties: props, } w.indexSepAlloc, part.sep = cloneKeyWithBuf( - base.DecodeInternalKey(indexBuf.block.curKey), w.indexSepAlloc, + indexBuf.block.getCurKey(), w.indexSepAlloc, ) bk := indexBuf.finish() if len(w.indexBlockAlloc) < len(bk) { @@ -1660,7 +1775,7 @@ func compressAndChecksum(b []byte, compression Compression, blockBuf *blockBuf) func (w *Writer) writeCompressedBlock(block []byte, blockTrailerBuf []byte) (BlockHandle, error) { bh := BlockHandle{Offset: w.meta.Size, Length: uint64(len(block))} - if w.cacheID != 0 && w.fileNum != 0 { + if w.cacheID != 0 && w.fileNum.FileNum() != 0 { // Remove the block being written from the cache. This provides defense in // depth against bugs which cause cache collisions. // @@ -1687,7 +1802,7 @@ func (w *Writer) writeCompressedBlock(block []byte, blockTrailerBuf []byte) (Blo // return a BlockHandle. func (w *Writer) Write(blockWithTrailer []byte) (n int, err error) { offset := w.meta.Size - if w.cacheID != 0 && w.fileNum != 0 { + if w.cacheID != 0 && w.fileNum.FileNum() != 0 { // Remove the block being written from the cache. This provides defense in // depth against bugs which cause cache collisions. // @@ -1728,12 +1843,20 @@ func (w *Writer) assertFormatCompatibility() error { ) } + // PebbleDBv3: value blocks. if (w.props.NumValueBlocks > 0 || w.props.NumValuesInValueBlocks > 0 || w.props.ValueBlocksSize > 0) && w.tableFormat < TableFormatPebblev3 { return errors.Newf( "table format version %s is less than the minimum required version %s for value blocks", w.tableFormat, TableFormatPebblev3) } + + // PebbleDBv4: DELSIZED tombstones. + if w.props.NumSizedDeletions > 0 && w.tableFormat < TableFormatPebblev4 { + return errors.Newf( + "table format version %s is less than the minimum required version %s for sized deletion tombstones", + w.tableFormat, TableFormatPebblev4) + } return nil } @@ -1780,7 +1903,7 @@ func (w *Writer) Close() (err error) { // however, if a dataBlock is flushed, then we add a key to the new w.dataBlockBuf in the // addPoint function after the flush occurs. if w.dataBlockBuf.dataBlock.nEntries >= 1 { - w.meta.SetLargestPointKey(base.DecodeInternalKey(w.dataBlockBuf.dataBlock.curKey).Clone()) + w.meta.SetLargestPointKey(w.dataBlockBuf.dataBlock.getCurKey().Clone()) } // Finish the last data block, or force an empty data block if there @@ -1794,7 +1917,7 @@ func (w *Writer) Close() (err error) { if err != nil { return err } - prevKey := base.DecodeInternalKey(w.dataBlockBuf.dataBlock.curKey) + prevKey := w.dataBlockBuf.dataBlock.getCurKey() if err := w.addIndexEntrySync(prevKey, InternalKey{}, bhp, w.dataBlockBuf.tmp[:]); err != nil { return err } @@ -1873,7 +1996,7 @@ func (w *Writer) Close() (err error) { var rangeKeyBH BlockHandle if w.props.NumRangeKeys() > 0 { - key := base.DecodeInternalKey(w.rangeKeyBlock.curKey) + key := w.rangeKeyBlock.getCurKey() kind := key.Kind() endKey, _, ok := rangekey.DecodeEndKey(kind, w.rangeKeyBlock.curValue) if !ok { @@ -1948,7 +2071,7 @@ func (w *Writer) Close() (err error) { // reduces table size without a significant impact on performance. raw.restartInterval = propertiesBlockRestartInterval w.props.CompressionOptions = rocksDBCompressionOptions - w.props.save(&raw) + w.props.save(w.tableFormat, &raw) bh, err := w.writeBlock(raw.finish(), NoCompression, &w.blockBuf) if err != nil { return err @@ -2062,7 +2185,7 @@ func (o PreviousPointKeyOpt) UnsafeKey() base.InternalKey { if o.w.dataBlockBuf.dataBlock.nEntries >= 1 { // o.w.dataBlockBuf.dataBlock.curKey is guaranteed to point to the last point key // which was added to the Writer. - return base.DecodeInternalKey(o.w.dataBlockBuf.dataBlock.curKey) + return o.w.dataBlockBuf.dataBlock.getCurKey() } return base.InternalKey{} } @@ -2091,6 +2214,8 @@ func NewWriter(writable objstorage.Writable, o WriterOptions, extraOpts ...Write separator: o.Comparer.Separator, successor: o.Comparer.Successor, tableFormat: o.TableFormat, + isStrictObsolete: o.IsStrictObsolete, + writingToLowestLevel: o.WritingToLowestLevel, cache: o.Cache, restartInterval: o.BlockRestartInterval, checksumType: o.Checksum, @@ -2109,7 +2234,7 @@ func NewWriter(writable objstorage.Writable, o WriterOptions, extraOpts ...Write Format: o.Comparer.FormatKey, }, } - if w.tableFormat == TableFormatPebblev3 { + if w.tableFormat >= TableFormatPebblev3 { w.shortAttributeExtractor = o.ShortAttributeExtractor w.requiredInPlaceValueBound = o.RequiredInPlaceValueBound w.valueBlockWriter = newValueBlockWriter( @@ -2157,14 +2282,14 @@ func NewWriter(writable objstorage.Writable, o WriterOptions, extraOpts ...Write } } - w.props.ColumnFamilyID = math.MaxInt32 w.props.ComparerName = o.Comparer.Name w.props.CompressionName = o.Compression.String() w.props.MergerName = o.MergerName w.props.PropertyCollectorNames = "[]" w.props.ExternalFormatVersion = rocksDBExternalFormatVersion - if len(o.TablePropertyCollectors) > 0 || len(o.BlockPropertyCollectors) > 0 { + if len(o.TablePropertyCollectors) > 0 || len(o.BlockPropertyCollectors) > 0 || + w.tableFormat >= TableFormatPebblev4 { var buf bytes.Buffer buf.WriteString("[") if len(o.TablePropertyCollectors) > 0 { @@ -2177,16 +2302,22 @@ func NewWriter(writable objstorage.Writable, o WriterOptions, extraOpts ...Write buf.WriteString(w.propCollectors[i].Name()) } } + numBlockPropertyCollectors := len(o.BlockPropertyCollectors) + if w.tableFormat >= TableFormatPebblev4 { + numBlockPropertyCollectors++ + } + // shortID is a uint8, so we cannot exceed that number of block + // property collectors. + if numBlockPropertyCollectors > math.MaxUint8 { + w.err = errors.New("pebble: too many block property collectors") + return w + } + if numBlockPropertyCollectors > 0 { + w.blockPropCollectors = make([]BlockPropertyCollector, numBlockPropertyCollectors) + } if len(o.BlockPropertyCollectors) > 0 { - // shortID is a uint8, so we cannot exceed that number of block - // property collectors. - if len(o.BlockPropertyCollectors) > math.MaxUint8 { - w.err = errors.New("pebble: too many block property collectors") - return w - } // The shortID assigned to a collector is the same as its index in // this slice. - w.blockPropCollectors = make([]BlockPropertyCollector, len(o.BlockPropertyCollectors)) for i := range o.BlockPropertyCollectors { w.blockPropCollectors[i] = o.BlockPropertyCollectors[i]() if i > 0 || len(o.TablePropertyCollectors) > 0 { @@ -2195,6 +2326,13 @@ func NewWriter(writable objstorage.Writable, o WriterOptions, extraOpts ...Write buf.WriteString(w.blockPropCollectors[i].Name()) } } + if w.tableFormat >= TableFormatPebblev4 { + if numBlockPropertyCollectors > 1 || len(o.TablePropertyCollectors) > 0 { + buf.WriteString(",") + } + w.blockPropCollectors[numBlockPropertyCollectors-1] = &w.obsoleteCollector + buf.WriteString(w.obsoleteCollector.Name()) + } buf.WriteString("]") w.props.PropertyCollectorNames = buf.String() } @@ -2228,3 +2366,85 @@ func init() { } private.SSTableInternalProperties = internalGetProperties } + +type obsoleteKeyBlockPropertyCollector struct { + blockIsNonObsolete bool + indexIsNonObsolete bool + tableIsNonObsolete bool +} + +func encodeNonObsolete(isNonObsolete bool, buf []byte) []byte { + if isNonObsolete { + return buf + } + return append(buf, 't') +} + +func (o *obsoleteKeyBlockPropertyCollector) Name() string { + return "obsolete-key" +} + +func (o *obsoleteKeyBlockPropertyCollector) Add(key InternalKey, value []byte) error { + // Ignore. + return nil +} + +func (o *obsoleteKeyBlockPropertyCollector) AddPoint(isObsolete bool) { + o.blockIsNonObsolete = o.blockIsNonObsolete || !isObsolete +} + +func (o *obsoleteKeyBlockPropertyCollector) FinishDataBlock(buf []byte) ([]byte, error) { + o.tableIsNonObsolete = o.tableIsNonObsolete || o.blockIsNonObsolete + return encodeNonObsolete(o.blockIsNonObsolete, buf), nil +} + +func (o *obsoleteKeyBlockPropertyCollector) AddPrevDataBlockToIndexBlock() { + o.indexIsNonObsolete = o.indexIsNonObsolete || o.blockIsNonObsolete + o.blockIsNonObsolete = false +} + +func (o *obsoleteKeyBlockPropertyCollector) FinishIndexBlock(buf []byte) ([]byte, error) { + indexIsNonObsolete := o.indexIsNonObsolete + o.indexIsNonObsolete = false + return encodeNonObsolete(indexIsNonObsolete, buf), nil +} + +func (o *obsoleteKeyBlockPropertyCollector) FinishTable(buf []byte) ([]byte, error) { + return encodeNonObsolete(o.tableIsNonObsolete, buf), nil +} + +func (o *obsoleteKeyBlockPropertyCollector) UpdateKeySuffixes( + oldProp []byte, oldSuffix, newSuffix []byte, +) error { + _, err := propToIsObsolete(oldProp) + if err != nil { + return err + } + // Suffix rewriting currently loses the obsolete bit. + o.blockIsNonObsolete = true + return nil +} + +// NB: obsoleteKeyBlockPropertyFilter is stateless. This aspect of the filter +// is used in table_cache.go for in-place modification of a filters slice. +type obsoleteKeyBlockPropertyFilter struct{} + +func (o obsoleteKeyBlockPropertyFilter) Name() string { + return "obsolete-key" +} + +// Intersects returns true if the set represented by prop intersects with +// the set in the filter. +func (o obsoleteKeyBlockPropertyFilter) Intersects(prop []byte) (bool, error) { + return propToIsObsolete(prop) +} + +func propToIsObsolete(prop []byte) (bool, error) { + if len(prop) == 0 { + return true, nil + } + if len(prop) > 1 || prop[0] != 't' { + return false, errors.Errorf("unexpected property %x", prop) + } + return false, nil +} diff --git a/vendor/github.com/cockroachdb/pebble/table_cache.go b/vendor/github.com/cockroachdb/pebble/table_cache.go index 9add2b3..516e5e8 100644 --- a/vendor/github.com/cockroachdb/pebble/table_cache.go +++ b/vendor/github.com/cockroachdb/pebble/table_cache.go @@ -22,6 +22,7 @@ import ( "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/internal/private" "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing" "github.com/cockroachdb/pebble/sstable" ) @@ -58,18 +59,15 @@ var tableCacheLabels = pprof.Labels("pebble", "table-cache") // are updated, we could have unnecessary evictions of those // fields, and the surrounding fields from the CPU caches. type tableCacheOpts struct { - atomic struct { - // iterCount in the tableCacheOpts keeps track of iterators - // opened or closed by a DB. It's used to keep track of - // leaked iterators on a per-db level. - iterCount *int32 - } + // iterCount keeps track of how many iterators are open. It is used to keep + // track of leaked iterators on a per-db level. + iterCount *atomic.Int32 loggerAndTracer LoggerAndTracer cacheID uint64 - objProvider *objstorage.Provider + objProvider objstorage.Provider opts sstable.ReaderOptions - filterMetrics *FilterMetrics + filterMetrics *sstable.FilterMetricsTracker } // tableCacheContainer contains the table cache and @@ -85,7 +83,7 @@ type tableCacheContainer struct { // newTableCacheContainer will panic if the underlying cache in the table cache // doesn't match Options.Cache. func newTableCacheContainer( - tc *TableCache, cacheID uint64, objProvider *objstorage.Provider, opts *Options, size int, + tc *TableCache, cacheID uint64, objProvider objstorage.Provider, opts *Options, size int, ) *tableCacheContainer { // We will release a ref to table cache acquired here when tableCacheContainer.close is called. if tc != nil { @@ -105,8 +103,8 @@ func newTableCacheContainer( t.dbOpts.cacheID = cacheID t.dbOpts.objProvider = objProvider t.dbOpts.opts = opts.MakeReaderOptions() - t.dbOpts.filterMetrics = &FilterMetrics{} - t.dbOpts.atomic.iterCount = new(int32) + t.dbOpts.filterMetrics = &sstable.FilterMetricsTracker{} + t.dbOpts.iterCount = new(atomic.Int32) return t } @@ -117,7 +115,7 @@ func (c *tableCacheContainer) close() error { // by the DB using this container. Note that we'll still perform cleanup // below in the case that there are leaked iterators. var err error - if v := atomic.LoadInt32(c.dbOpts.atomic.iterCount); v > 0 { + if v := c.dbOpts.iterCount.Load(); v > 0 { err = errors.Errorf("leaked iterators: %d", errors.Safe(v)) } @@ -136,20 +134,22 @@ func (c *tableCacheContainer) newIters( opts *IterOptions, internalOpts internalIterOpts, ) (internalIterator, keyspan.FragmentIterator, error) { - return c.tableCache.getShard(file.FileNum).newIters(ctx, file, opts, internalOpts, &c.dbOpts) + return c.tableCache.getShard(file.FileBacking.DiskFileNum).newIters(ctx, file, opts, internalOpts, &c.dbOpts) } func (c *tableCacheContainer) newRangeKeyIter( - file *manifest.FileMetadata, opts *keyspan.SpanIterOptions, + file *manifest.FileMetadata, opts keyspan.SpanIterOptions, ) (keyspan.FragmentIterator, error) { - return c.tableCache.getShard(file.FileNum).newRangeKeyIter(file, opts, &c.dbOpts) + return c.tableCache.getShard(file.FileBacking.DiskFileNum).newRangeKeyIter(file, opts, &c.dbOpts) } +// getTableProperties returns the properties associated with the backing physical +// table if the input metadata belongs to a virtual sstable. func (c *tableCacheContainer) getTableProperties(file *fileMetadata) (*sstable.Properties, error) { - return c.tableCache.getShard(file.FileNum).getTableProperties(file, &c.dbOpts) + return c.tableCache.getShard(file.FileBacking.DiskFileNum).getTableProperties(file, &c.dbOpts) } -func (c *tableCacheContainer) evict(fileNum FileNum) { +func (c *tableCacheContainer) evict(fileNum base.DiskFileNum) { c.tableCache.getShard(fileNum).evict(fileNum, &c.dbOpts, false) } @@ -160,41 +160,109 @@ func (c *tableCacheContainer) metrics() (CacheMetrics, FilterMetrics) { s.mu.RLock() m.Count += int64(len(s.mu.nodes)) s.mu.RUnlock() - m.Hits += atomic.LoadInt64(&s.atomic.hits) - m.Misses += atomic.LoadInt64(&s.atomic.misses) + m.Hits += s.hits.Load() + m.Misses += s.misses.Load() } m.Size = m.Count * int64(unsafe.Sizeof(sstable.Reader{})) - f := FilterMetrics{ - Hits: atomic.LoadInt64(&c.dbOpts.filterMetrics.Hits), - Misses: atomic.LoadInt64(&c.dbOpts.filterMetrics.Misses), - } + f := c.dbOpts.filterMetrics.Load() return m, f } -func (c *tableCacheContainer) withReader(meta *fileMetadata, fn func(*sstable.Reader) error) error { - s := c.tableCache.getShard(meta.FileNum) +func (c *tableCacheContainer) estimateSize( + meta *fileMetadata, lower, upper []byte, +) (size uint64, err error) { + if meta.Virtual { + err = c.withVirtualReader( + meta.VirtualMeta(), + func(r sstable.VirtualReader) (err error) { + size, err = r.EstimateDiskUsage(lower, upper) + return err + }, + ) + } else { + err = c.withReader( + meta.PhysicalMeta(), + func(r *sstable.Reader) (err error) { + size, err = r.EstimateDiskUsage(lower, upper) + return err + }, + ) + } + if err != nil { + return 0, err + } + return size, nil +} + +// createCommonReader creates a Reader for this file. isForeign, if true for +// virtual sstables, is passed into the vSSTable reader so its iterators can +// collapse obsolete points accordingly. +func createCommonReader( + v *tableCacheValue, file *fileMetadata, isForeign bool, +) sstable.CommonReader { + // TODO(bananabrick): We suffer an allocation if file is a virtual sstable. + var cr sstable.CommonReader = v.reader + if file.Virtual { + virtualReader := sstable.MakeVirtualReader( + v.reader, file.VirtualMeta(), isForeign, + ) + cr = &virtualReader + } + return cr +} + +func (c *tableCacheContainer) withCommonReader( + meta *fileMetadata, fn func(sstable.CommonReader) error, +) error { + s := c.tableCache.getShard(meta.FileBacking.DiskFileNum) v := s.findNode(meta, &c.dbOpts) defer s.unrefValue(v) if v.err != nil { return v.err } + provider := c.dbOpts.objProvider + objMeta, err := provider.Lookup(fileTypeTable, meta.FileBacking.DiskFileNum) + if err != nil { + return err + } + return fn(createCommonReader(v, meta, provider.IsSharedForeign(objMeta))) +} + +func (c *tableCacheContainer) withReader(meta physicalMeta, fn func(*sstable.Reader) error) error { + s := c.tableCache.getShard(meta.FileBacking.DiskFileNum) + v := s.findNode(meta.FileMetadata, &c.dbOpts) + defer s.unrefValue(v) + if v.err != nil { + return v.err + } return fn(v.reader) } +// withVirtualReader fetches a VirtualReader associated with a virtual sstable. +func (c *tableCacheContainer) withVirtualReader( + meta virtualMeta, fn func(sstable.VirtualReader) error, +) error { + s := c.tableCache.getShard(meta.FileBacking.DiskFileNum) + v := s.findNode(meta.FileMetadata, &c.dbOpts) + defer s.unrefValue(v) + if v.err != nil { + return v.err + } + provider := c.dbOpts.objProvider + objMeta, err := provider.Lookup(fileTypeTable, meta.FileBacking.DiskFileNum) + if err != nil { + return err + } + return fn(sstable.MakeVirtualReader(v.reader, meta, provider.IsSharedForeign(objMeta))) +} + func (c *tableCacheContainer) iterCount() int64 { - return int64(atomic.LoadInt32(c.dbOpts.atomic.iterCount)) + return int64(c.dbOpts.iterCount.Load()) } // TableCache is a shareable cache for open sstables. type TableCache struct { - // atomic contains fields which are accessed atomically. Go allocations - // are guaranteed to be 64-bit aligned which we take advantage of by - // placing the 64-bit fields which we access atomically at the beginning - // of the TableCache struct. For more information, see - // https://golang.org/pkg/sync/atomic/#pkg-note-BUG. - atomic struct { - refs int64 - } + refs atomic.Int64 cache *Cache shards []*tableCacheShard @@ -204,7 +272,7 @@ type TableCache struct { // the table cache only remains valid if there is at least one reference // to it. func (c *TableCache) Ref() { - v := atomic.AddInt64(&c.atomic.refs, 1) + v := c.refs.Add(1) // We don't want the reference count to ever go from 0 -> 1, // cause a reference count of 0 implies that we've closed the cache. if v <= 1 { @@ -214,7 +282,7 @@ func (c *TableCache) Ref() { // Unref removes a reference to the table cache. func (c *TableCache) Unref() error { - v := atomic.AddInt64(&c.atomic.refs, -1) + v := c.refs.Add(-1) switch { case v < 0: panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v)) @@ -256,31 +324,24 @@ func NewTableCache(cache *Cache, numShards int, size int) *TableCache { } // Hold a ref to the cache here. - c.atomic.refs = 1 + c.refs.Store(1) return c } -func (c *TableCache) getShard(fileNum FileNum) *tableCacheShard { - return c.shards[uint64(fileNum)%uint64(len(c.shards))] +func (c *TableCache) getShard(fileNum base.DiskFileNum) *tableCacheShard { + return c.shards[uint64(fileNum.FileNum())%uint64(len(c.shards))] } type tableCacheKey struct { cacheID uint64 - fileNum FileNum + fileNum base.DiskFileNum } type tableCacheShard struct { - // WARNING: The following struct `atomic` contains fields are accessed atomically. - // - // Go allocations are guaranteed to be 64-bit aligned which we take advantage - // of by placing the 64-bit fields which we access atomically at the beginning - // of the DB struct. For more information, see https://golang.org/pkg/sync/atomic/#pkg-note-BUG. - atomic struct { - hits int64 - misses int64 - iterCount int32 - } + hits atomic.Int64 + misses atomic.Int64 + iterCount atomic.Int32 size int @@ -377,21 +438,52 @@ func (c *tableCacheShard) newIters( return nil, nil, v.err } + hideObsoletePoints := false + var pointKeyFilters []BlockPropertyFilter + if opts != nil { + // This code is appending (at most one filter) in-place to + // opts.PointKeyFilters even though the slice is shared for iterators in + // the same iterator tree. This is acceptable since all the following + // properties are true: + // - The iterator tree is single threaded, so the shared backing for the + // slice is being mutated in a single threaded manner. + // - Each shallow copy of the slice has its own notion of length. + // - The appended element is always the obsoleteKeyBlockPropertyFilter + // struct, which is stateless, so overwriting that struct when creating + // one sstable iterator is harmless to other sstable iterators that are + // relying on that struct. + // + // An alternative would be to have different slices for different sstable + // iterators, but that requires more work to avoid allocations. + hideObsoletePoints, pointKeyFilters = + v.reader.TryAddBlockPropertyFilterForHideObsoletePoints( + opts.snapshotForHideObsoletePoints, file.LargestSeqNum, opts.PointKeyFilters) + } ok := true var filterer *sstable.BlockPropertiesFilterer var err error if opts != nil { ok, filterer, err = c.checkAndIntersectFilters(v, opts.TableFilter, - opts.PointKeyFilters, internalOpts.boundLimitedFilter) + pointKeyFilters, internalOpts.boundLimitedFilter) } if err != nil { c.unrefValue(v) return nil, nil, err } + provider := dbOpts.objProvider + // Check if this file is a foreign file. + objMeta, err := provider.Lookup(fileTypeTable, file.FileBacking.DiskFileNum) + if err != nil { + return nil, nil, err + } + + // Note: This suffers an allocation for virtual sstables. + cr := createCommonReader(v, file, provider.IsSharedForeign(objMeta)) + // NB: range-del iterator does not maintain a reference to the table, nor // does it need to read from it after creation. - rangeDelIter, err := v.reader.NewRawRangeDelIter() + rangeDelIter, err := cr.NewRawRangeDelIter() if err != nil { c.unrefValue(v) return nil, nil, err @@ -417,20 +509,29 @@ func (c *tableCacheShard) newIters( useFilter := true if opts != nil { useFilter = manifest.LevelToInt(opts.level) != 6 || opts.UseL6Filters + ctx = objiotracing.WithLevel(ctx, manifest.LevelToInt(opts.level)) } tableFormat, err := v.reader.TableFormat() if err != nil { return nil, nil, err } var rp sstable.ReaderProvider - if tableFormat == sstable.TableFormatPebblev3 && v.reader.Properties.NumValueBlocks > 0 { + if tableFormat >= sstable.TableFormatPebblev3 && v.reader.Properties.NumValueBlocks > 0 { rp = &tableCacheShardReaderProvider{c: c, file: file, dbOpts: dbOpts} } + + if provider.IsSharedForeign(objMeta) { + if tableFormat < sstable.TableFormatPebblev4 { + return nil, nil, errors.New("pebble: shared foreign sstable has a lower table format than expected") + } + hideObsoletePoints = true + } if internalOpts.bytesIterated != nil { - iter, err = v.reader.NewCompactionIter(internalOpts.bytesIterated, rp) + iter, err = cr.NewCompactionIter(internalOpts.bytesIterated, rp, internalOpts.bufferPool) } else { - iter, err = v.reader.NewIterWithBlockPropertyFiltersAndContext( - ctx, opts.GetLowerBound(), opts.GetUpperBound(), filterer, useFilter, internalOpts.stats, rp) + iter, err = cr.NewIterWithBlockPropertyFiltersAndContextEtc( + ctx, opts.GetLowerBound(), opts.GetUpperBound(), filterer, hideObsoletePoints, useFilter, + internalOpts.stats, rp) } if err != nil { if rangeDelIter != nil { @@ -443,8 +544,8 @@ func (c *tableCacheShard) newIters( // care to avoid introducing an allocation here by adding a closure. iter.SetCloseHook(v.closeHook) - atomic.AddInt32(&c.atomic.iterCount, 1) - atomic.AddInt32(dbOpts.atomic.iterCount, 1) + c.iterCount.Add(1) + dbOpts.iterCount.Add(1) if invariants.RaceEnabled { c.mu.Lock() c.mu.iters[iter] = debug.Stack() @@ -454,7 +555,7 @@ func (c *tableCacheShard) newIters( } func (c *tableCacheShard) newRangeKeyIter( - file *manifest.FileMetadata, opts *keyspan.SpanIterOptions, dbOpts *tableCacheOpts, + file *manifest.FileMetadata, opts keyspan.SpanIterOptions, dbOpts *tableCacheOpts, ) (keyspan.FragmentIterator, error) { // Calling findNode gives us the responsibility of decrementing v's // refCount. If opening the underlying table resulted in error, then we @@ -473,7 +574,7 @@ func (c *tableCacheShard) newRangeKeyIter( // file's range key blocks may surface deleted range keys below. This is // done here, rather than deferring to the block-property collector in order // to maintain parity with point keys and the treatment of RANGEDELs. - if opts != nil && v.reader.Properties.NumRangeKeyDels == 0 { + if v.reader.Properties.NumRangeKeyDels == 0 { ok, _, err = c.checkAndIntersectFilters(v, nil, opts.RangeKeyFilters, nil) } if err != nil { @@ -488,7 +589,20 @@ func (c *tableCacheShard) newRangeKeyIter( } var iter keyspan.FragmentIterator - iter, err = v.reader.NewRawRangeKeyIter() + if file.Virtual { + provider := dbOpts.objProvider + var objMeta objstorage.ObjectMetadata + objMeta, err = provider.Lookup(fileTypeTable, file.FileBacking.DiskFileNum) + if err == nil { + virtualReader := sstable.MakeVirtualReader( + v.reader, file.VirtualMeta(), provider.IsSharedForeign(objMeta), + ) + iter, err = virtualReader.NewRawRangeKeyIter() + } + } else { + iter, err = v.reader.NewRawRangeKeyIter() + } + // iter is a block iter that holds the entire value of the block in memory. // No need to hold onto a ref of the cache value. c.unrefValue(v) @@ -522,6 +636,13 @@ var _ sstable.ReaderProvider = &tableCacheShardReaderProvider{} // being deleted. // // The caller must call tableCacheShardReaderProvider.Close. +// +// Note that currently the Reader returned here is only used to read value +// blocks. This reader shouldn't be used for other purposes like reading keys +// outside of virtual sstable bounds. +// +// TODO(bananabrick): We could return a wrapper over the Reader to ensure +// that the reader isn't used for other purposes. func (rp *tableCacheShardReaderProvider) GetReader() (*sstable.Reader, error) { // Calling findNode gives us the responsibility of decrementing v's // refCount. @@ -567,7 +688,7 @@ func (c *tableCacheShard) releaseNode(n *tableCacheNode) { // // c.mu must be held when calling this. func (c *tableCacheShard) unlinkNode(n *tableCacheNode) { - key := tableCacheKey{n.cacheID, n.meta.FileNum} + key := tableCacheKey{n.cacheID, n.fileNum} delete(c.mu.nodes, key) switch n.ptype { @@ -612,7 +733,7 @@ func (c *tableCacheShard) clearNode(n *tableCacheNode) { // it is present in tableCacheShard.mu.nodes, so a reference count of 0 means // the node has already been removed from that map. func (c *tableCacheShard) unrefValue(v *tableCacheValue) { - if atomic.AddInt32(&v.refCount, -1) == 0 { + if v.refCount.Add(-1) == 0 { c.releasing.Add(1) c.releasingCh <- v } @@ -622,18 +743,41 @@ func (c *tableCacheShard) unrefValue(v *tableCacheValue) { // that node if it didn't already exist. The caller is responsible for // decrementing the returned node's refCount. func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) *tableCacheValue { + v := c.findNodeInternal(meta, dbOpts) + + // Loading a file before its global sequence number is known (eg, + // during ingest before entering the commit pipeline) can pollute + // the cache with incorrect state. In invariant builds, verify + // that the global sequence number of the returned reader matches. + if invariants.Enabled { + if v.reader != nil && meta.LargestSeqNum == meta.SmallestSeqNum && + v.reader.Properties.GlobalSeqNum != meta.SmallestSeqNum { + panic(errors.AssertionFailedf("file %s loaded from table cache with the wrong global sequence number %d", + meta, v.reader.Properties.GlobalSeqNum)) + } + } + return v +} + +func (c *tableCacheShard) findNodeInternal( + meta *fileMetadata, dbOpts *tableCacheOpts, +) *tableCacheValue { + if refs := meta.Refs(); refs <= 0 { + panic(errors.AssertionFailedf("attempting to load file %s with refs=%d from table cache", + meta, refs)) + } // Fast-path for a hit in the cache. c.mu.RLock() - key := tableCacheKey{dbOpts.cacheID, meta.FileNum} + key := tableCacheKey{dbOpts.cacheID, meta.FileBacking.DiskFileNum} if n := c.mu.nodes[key]; n != nil && n.value != nil { // Fast-path hit. // // The caller is responsible for decrementing the refCount. v := n.value - atomic.AddInt32(&v.refCount, 1) + v.refCount.Add(1) c.mu.RUnlock() - atomic.StoreInt32(&n.referenced, 1) - atomic.AddInt64(&c.atomic.hits, 1) + n.referenced.Store(true) + c.hits.Add(1) <-v.loaded return v } @@ -646,8 +790,8 @@ func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) * case n == nil: // Slow-path miss of a non-existent node. n = &tableCacheNode{ - meta: meta, - ptype: tableCacheNodeCold, + fileNum: meta.FileBacking.DiskFileNum, + ptype: tableCacheNodeCold, } c.addNode(n, dbOpts) c.mu.sizeCold++ @@ -657,9 +801,9 @@ func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) * // // The caller is responsible for decrementing the refCount. v := n.value - atomic.AddInt32(&v.refCount, 1) - atomic.StoreInt32(&n.referenced, 1) - atomic.AddInt64(&c.atomic.hits, 1) + v.refCount.Add(1) + n.referenced.Store(true) + c.hits.Add(1) c.mu.Unlock() <-v.loaded return v @@ -672,18 +816,18 @@ func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) * c.mu.coldTarget = c.size } - atomic.StoreInt32(&n.referenced, 0) + n.referenced.Store(false) n.ptype = tableCacheNodeHot c.addNode(n, dbOpts) c.mu.sizeHot++ } - atomic.AddInt64(&c.atomic.misses, 1) + c.misses.Add(1) v := &tableCacheValue{ - loaded: make(chan struct{}), - refCount: 2, + loaded: make(chan struct{}), } + v.refCount.Store(2) // Cache the closure invoked when an iterator is closed. This avoids an // allocation on every call to newIters. v.closeHook = func(i sstable.Iterator) error { @@ -693,8 +837,8 @@ func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) * c.mu.Unlock() } c.unrefValue(v) - atomic.AddInt32(&c.atomic.iterCount, -1) - atomic.AddInt32(dbOpts.atomic.iterCount, -1) + c.iterCount.Add(-1) + dbOpts.iterCount.Add(-1) return nil } n.value = v @@ -704,7 +848,12 @@ func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) * // Note adding to the cache lists must complete before we begin loading the // table as a failure during load will result in the node being unlinked. pprof.Do(context.Background(), tableCacheLabels, func(context.Context) { - v.load(meta, c, dbOpts) + v.load( + loadInfo{ + backingFileNum: meta.FileBacking.DiskFileNum, + smallestSeqNum: meta.SmallestSeqNum, + largestSeqNum: meta.LargestSeqNum, + }, c, dbOpts) }) return v } @@ -712,7 +861,7 @@ func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) * func (c *tableCacheShard) addNode(n *tableCacheNode, dbOpts *tableCacheOpts) { c.evictNodes() n.cacheID = dbOpts.cacheID - key := tableCacheKey{n.cacheID, n.meta.FileNum} + key := tableCacheKey{n.cacheID, n.fileNum} c.mu.nodes[key] = n n.links.next = n @@ -740,8 +889,8 @@ func (c *tableCacheShard) evictNodes() { func (c *tableCacheShard) runHandCold() { n := c.mu.handCold if n.ptype == tableCacheNodeCold { - if atomic.LoadInt32(&n.referenced) == 1 { - atomic.StoreInt32(&n.referenced, 0) + if n.referenced.Load() { + n.referenced.Store(false) n.ptype = tableCacheNodeHot c.mu.sizeCold-- c.mu.sizeHot++ @@ -773,8 +922,8 @@ func (c *tableCacheShard) runHandHot() { n := c.mu.handHot if n.ptype == tableCacheNodeHot { - if atomic.LoadInt32(&n.referenced) == 1 { - atomic.StoreInt32(&n.referenced, 0) + if n.referenced.Load() { + n.referenced.Store(false) } else { n.ptype = tableCacheNodeCold c.mu.sizeHot-- @@ -806,7 +955,7 @@ func (c *tableCacheShard) runHandTest() { c.mu.handTest = c.mu.handTest.next() } -func (c *tableCacheShard) evict(fileNum FileNum, dbOpts *tableCacheOpts, allowLeak bool) { +func (c *tableCacheShard) evict(fileNum base.DiskFileNum, dbOpts *tableCacheOpts, allowLeak bool) { c.mu.Lock() key := tableCacheKey{dbOpts.cacheID, fileNum} n := c.mu.nodes[key] @@ -821,7 +970,7 @@ func (c *tableCacheShard) evict(fileNum FileNum, dbOpts *tableCacheOpts, allowLe v = n.value if v != nil { if !allowLeak { - if t := atomic.AddInt32(&v.refCount, -1); t != 0 { + if t := v.refCount.Add(-1); t != 0 { dbOpts.loggerAndTracer.Fatalf("sstable %s: refcount is not zero: %d\n%s", fileNum, t, debug.Stack()) } } @@ -842,7 +991,7 @@ func (c *tableCacheShard) evict(fileNum FileNum, dbOpts *tableCacheOpts, allowLe // associated with dbOpts.cacheID. Make sure that there will // be no more accesses to the files associated with the DB. func (c *tableCacheShard) removeDB(dbOpts *tableCacheOpts) { - var fileNums []base.FileNum + var fileNums []base.DiskFileNum c.mu.RLock() // Collect the fileNums which need to be cleaned. @@ -854,7 +1003,7 @@ func (c *tableCacheShard) removeDB(dbOpts *tableCacheOpts) { } if node.cacheID == dbOpts.cacheID { - fileNums = append(fileNums, node.meta.FileNum) + fileNums = append(fileNums, node.fileNum) } node = node.next() } @@ -863,8 +1012,8 @@ func (c *tableCacheShard) removeDB(dbOpts *tableCacheOpts) { // Evict all the nodes associated with the DB. // This should synchronously close all the files // associated with the DB. - for _, fNum := range fileNums { - c.evict(fNum, dbOpts, true) + for _, fileNum := range fileNums { + c.evict(fileNum, dbOpts, true) } } @@ -875,7 +1024,7 @@ func (c *tableCacheShard) Close() error { // Check for leaked iterators. Note that we'll still perform cleanup below in // the case that there are leaked iterators. var err error - if v := atomic.LoadInt32(&c.atomic.iterCount); v > 0 { + if v := c.iterCount.Load(); v > 0 { if !invariants.RaceEnabled { err = errors.Errorf("leaked iterators: %d", errors.Safe(v)) } else { @@ -890,7 +1039,7 @@ func (c *tableCacheShard) Close() error { for c.mu.handHot != nil { n := c.mu.handHot if n.value != nil { - if atomic.AddInt32(&n.value.refCount, -1) == 0 { + if n.value.refCount.Add(-1) == 0 { c.releasing.Add(1) c.releasingCh <- n.value } @@ -926,31 +1075,39 @@ type tableCacheValue struct { loaded chan struct{} // Reference count for the value. The reader is closed when the reference // count drops to zero. - refCount int32 + refCount atomic.Int32 } -func (v *tableCacheValue) load(meta *fileMetadata, c *tableCacheShard, dbOpts *tableCacheOpts) { +type loadInfo struct { + backingFileNum base.DiskFileNum + largestSeqNum uint64 + smallestSeqNum uint64 +} + +func (v *tableCacheValue) load(loadInfo loadInfo, c *tableCacheShard, dbOpts *tableCacheOpts) { // Try opening the file first. var f objstorage.Readable - f, v.err = dbOpts.objProvider.OpenForReadingMustExist(fileTypeTable, meta.FileNum) - if v.err == nil { - cacheOpts := private.SSTableCacheOpts(dbOpts.cacheID, meta.FileNum).(sstable.ReaderOption) - v.reader, v.err = sstable.NewReader(f, dbOpts.opts, cacheOpts, dbOpts.filterMetrics) + var err error + f, err = dbOpts.objProvider.OpenForReading( + context.TODO(), fileTypeTable, loadInfo.backingFileNum, objstorage.OpenOptions{MustExist: true}, + ) + if err == nil { + cacheOpts := private.SSTableCacheOpts(dbOpts.cacheID, loadInfo.backingFileNum).(sstable.ReaderOption) + v.reader, err = sstable.NewReader(f, dbOpts.opts, cacheOpts, dbOpts.filterMetrics) } - if v.err != nil { - v.err = errors.Wrapf(v.err, "pebble: backing file %s error", errors.Safe(meta.FileNum)) + if err != nil { + v.err = errors.Wrapf( + err, "pebble: backing file %s error", errors.Safe(loadInfo.backingFileNum.FileNum())) } - if v.err == nil { - if meta.SmallestSeqNum == meta.LargestSeqNum { - v.reader.Properties.GlobalSeqNum = meta.LargestSeqNum - } + if v.err == nil && loadInfo.smallestSeqNum == loadInfo.largestSeqNum { + v.reader.Properties.GlobalSeqNum = loadInfo.largestSeqNum } if v.err != nil { c.mu.Lock() defer c.mu.Unlock() // Lookup the node in the cache again as it might have already been // removed. - key := tableCacheKey{dbOpts.cacheID, meta.FileNum} + key := tableCacheKey{dbOpts.cacheID, loadInfo.backingFileNum} n := c.mu.nodes[key] if n != nil && n.value == v { c.releaseNode(n) @@ -990,8 +1147,8 @@ func (p tableCacheNodeType) String() string { } type tableCacheNode struct { - meta *fileMetadata - value *tableCacheValue + fileNum base.DiskFileNum + value *tableCacheValue links struct { next *tableCacheNode @@ -1000,7 +1157,7 @@ type tableCacheNode struct { ptype tableCacheNodeType // referenced is atomically set to indicate that this entry has been accessed // since the last time one of the clock hands swept it. - referenced int32 + referenced atomic.Bool // Storing the cache id associated with the DB instance here // avoids the need to thread the dbOpts struct through many functions. diff --git a/vendor/github.com/cockroachdb/pebble/table_stats.go b/vendor/github.com/cockroachdb/pebble/table_stats.go index 7c025e9..f149f44 100644 --- a/vendor/github.com/cockroachdb/pebble/table_stats.go +++ b/vendor/github.com/cockroachdb/pebble/table_stats.go @@ -53,7 +53,7 @@ func (d *DB) maybeCollectTableStatsLocked() { func (d *DB) updateTableStatsLocked(newFiles []manifest.NewFileEntry) { var needStats bool for _, nf := range newFiles { - if !nf.Meta.StatsValidLocked() { + if !nf.Meta.StatsValid() { needStats = true break } @@ -175,7 +175,7 @@ func (d *DB) loadNewFileStats( // collectTableStats updates f.Stats for active files, and we // ensure only one goroutine runs it at a time through // d.mu.tableStats.loading. - if nf.Meta.StatsValidLocked() { + if nf.Meta.StatsValid() { continue } @@ -186,7 +186,10 @@ func (d *DB) loadNewFileStats( continue } - stats, newHints, err := d.loadTableStats(rs.current, nf.Level, nf.Meta) + stats, newHints, err := d.loadTableStats( + rs.current, nf.Level, + nf.Meta, + ) if err != nil { d.opts.EventListener.BackgroundError(err) continue @@ -219,7 +222,7 @@ func (d *DB) scanReadStateTableStats( // and we ensure only one goroutine runs it at a time through // d.mu.tableStats.loading. This makes it safe to read validity // through f.Stats.ValidLocked despite not holding d.mu. - if f.StatsValidLocked() { + if f.StatsValid() { continue } @@ -233,7 +236,9 @@ func (d *DB) scanReadStateTableStats( return fill, hints, moreRemain } - stats, newHints, err := d.loadTableStats(rs.current, l, f) + stats, newHints, err := d.loadTableStats( + rs.current, l, f, + ) if err != nil { // Set `moreRemain` so we'll try again. moreRemain = true @@ -255,26 +260,30 @@ func (d *DB) loadTableStats( ) (manifest.TableStats, []deleteCompactionHint, error) { var stats manifest.TableStats var compactionHints []deleteCompactionHint - err := d.tableCache.withReader(meta, func(r *sstable.Reader) (err error) { - stats.NumEntries = r.Properties.NumEntries - stats.NumDeletions = r.Properties.NumDeletions - if r.Properties.NumPointDeletions() > 0 { - if err = d.loadTablePointKeyStats(r, v, level, meta, &stats); err != nil { - return + err := d.tableCache.withCommonReader( + meta, func(r sstable.CommonReader) (err error) { + props := r.CommonProperties() + stats.NumEntries = props.NumEntries + stats.NumDeletions = props.NumDeletions + if props.NumPointDeletions() > 0 { + if err = d.loadTablePointKeyStats(props, v, level, meta, &stats); err != nil { + return + } } - } - if r.Properties.NumRangeDeletions > 0 || r.Properties.NumRangeKeyDels > 0 { - if compactionHints, err = d.loadTableRangeDelStats(r, v, level, meta, &stats); err != nil { - return + if props.NumRangeDeletions > 0 || props.NumRangeKeyDels > 0 { + if compactionHints, err = d.loadTableRangeDelStats( + r, v, level, meta, &stats, + ); err != nil { + return + } } - } - // TODO(travers): Once we have real-world data, consider collecting - // additional stats that may provide improved heuristics for compaction - // picking. - stats.NumRangeKeySets = r.Properties.NumRangeKeySets - stats.ValueBlocksSize = r.Properties.ValueBlocksSize - return - }) + // TODO(travers): Once we have real-world data, consider collecting + // additional stats that may provide improved heuristics for compaction + // picking. + stats.NumRangeKeySets = props.NumRangeKeySets + stats.ValueBlocksSize = props.ValueBlocksSize + return + }) if err != nil { return stats, nil, err } @@ -284,7 +293,11 @@ func (d *DB) loadTableStats( // loadTablePointKeyStats calculates the point key statistics for the given // table. The provided manifest.TableStats are updated. func (d *DB) loadTablePointKeyStats( - r *sstable.Reader, v *version, level int, meta *fileMetadata, stats *manifest.TableStats, + props *sstable.CommonProperties, + v *version, + level int, + meta *fileMetadata, + stats *manifest.TableStats, ) error { // TODO(jackson): If the file has a wide keyspace, the average // value size beneath the entire file might not be representative @@ -292,19 +305,19 @@ func (d *DB) loadTablePointKeyStats( // We could write the ranges of 'clusters' of point tombstones to // a sstable property and call averageValueSizeBeneath for each of // these narrower ranges to improve the estimate. - avgKeySize, avgValSize, err := d.averageEntrySizeBeneath(v, level, meta) + avgValLogicalSize, compressionRatio, err := d.estimateSizesBeneath(v, level, meta, props) if err != nil { return err } stats.PointDeletionsBytesEstimate = - pointDeletionsBytesEstimate(&r.Properties, avgKeySize, avgValSize) + pointDeletionsBytesEstimate(meta.Size, props, avgValLogicalSize, compressionRatio) return nil } // loadTableRangeDelStats calculates the range deletion and range key deletion // statistics for the given table. func (d *DB) loadTableRangeDelStats( - r *sstable.Reader, v *version, level int, meta *fileMetadata, stats *manifest.TableStats, + r sstable.CommonReader, v *version, level int, meta *fileMetadata, stats *manifest.TableStats, ) ([]deleteCompactionHint, error) { iter, err := newCombinedDeletionKeyspanIter(d.opts.Comparer, r, meta) if err != nil { @@ -320,25 +333,50 @@ func (d *DB) loadTableRangeDelStats( for s := iter.First(); s != nil; s = iter.Next() { start, end := s.Start, s.End // We only need to consider deletion size estimates for tables that contain - // point keys. - var hasPoints bool + // RANGEDELs. + var maxRangeDeleteSeqNum uint64 for _, k := range s.Keys { - if k.Kind() == base.InternalKeyKindRangeDelete { - hasPoints = true + if k.Kind() == base.InternalKeyKindRangeDelete && maxRangeDeleteSeqNum < k.SeqNum() { + maxRangeDeleteSeqNum = k.SeqNum() break } } // If the file is in the last level of the LSM, there is no data beneath // it. The fact that there is still a range tombstone in a bottommost file - // suggests that an open snapshot kept the tombstone around. Estimate disk - // usage within the file itself. + // indicates two possibilites: + // 1. an open snapshot kept the tombstone around, and the data the + // tombstone deletes is contained within the file itself. + // 2. the file was ingested. + // In the first case, we'd like to estimate disk usage within the file + // itself since compacting the file will drop that covered data. In the + // second case, we expect that compacting the file will NOT drop any + // data and rewriting the file is a waste of write bandwidth. We can + // distinguish these cases by looking at the file metadata's sequence + // numbers. A file's range deletions can only delete data within the + // file at lower sequence numbers. All keys in an ingested sstable adopt + // the same sequence number, preventing tombstones from deleting keys + // within the same file. We check here if the largest RANGEDEL sequence + // number is greater than the file's smallest sequence number. If it is, + // the RANGEDEL could conceivably (although inconclusively) delete data + // within the same file. + // + // Note that this heuristic is imperfect. If a table containing a range + // deletion is ingested into L5 and subsequently compacted into L6 but + // an open snapshot prevents elision of covered keys in L6, the + // resulting RangeDeletionsBytesEstimate will incorrectly include all + // covered keys. + // + // TODO(jackson): We could prevent the above error in the heuristic by + // computing the file's RangeDeletionsBytesEstimate during the + // compaction itself. It's unclear how common this is. + // // NOTE: If the span `s` wholly contains a table containing range keys, // the returned size estimate will be slightly inflated by the range key // block. However, in practice, range keys are expected to be rare, and // the size of the range key block relative to the overall size of the // table is expected to be small. - if hasPoints && level == numLevels-1 { + if level == numLevels-1 && meta.SmallestSeqNum < maxRangeDeleteSeqNum { size, err := r.EstimateDiskUsage(start, end) if err != nil { return nil, err @@ -383,24 +421,49 @@ func (d *DB) loadTableRangeDelStats( return compactionHints, err } -func (d *DB) averageEntrySizeBeneath( - v *version, level int, meta *fileMetadata, -) (avgKeySize, avgValueSize uint64, err error) { +func (d *DB) estimateSizesBeneath( + v *version, level int, meta *fileMetadata, fileProps *sstable.CommonProperties, +) (avgValueLogicalSize, compressionRatio float64, err error) { // Find all files in lower levels that overlap with meta, // summing their value sizes and entry counts. + file := meta var fileSum, keySum, valSum, entryCount uint64 + // Include the file itself. This is important because in some instances, the + // computed compression ratio is applied to the tombstones contained within + // `meta` itself. If there are no files beneath `meta` in the LSM, we would + // calculate a compression ratio of 0 which is not accurate for the file's + // own tombstones. + fileSum += file.Size + entryCount += fileProps.NumEntries + keySum += fileProps.RawKeySize + valSum += fileProps.RawValueSize + + addPhysicalTableStats := func(r *sstable.Reader) (err error) { + fileSum += file.Size + entryCount += r.Properties.NumEntries + keySum += r.Properties.RawKeySize + valSum += r.Properties.RawValueSize + return nil + } + addVirtualTableStats := func(v sstable.VirtualReader) (err error) { + fileSum += file.Size + entryCount += file.Stats.NumEntries + keySum += v.Properties.RawKeySize + valSum += v.Properties.RawValueSize + return nil + } + for l := level + 1; l < numLevels; l++ { overlaps := v.Overlaps(l, d.cmp, meta.Smallest.UserKey, meta.Largest.UserKey, meta.Largest.IsExclusiveSentinel()) iter := overlaps.Iter() - for file := iter.First(); file != nil; file = iter.Next() { - err := d.tableCache.withReader(file, func(r *sstable.Reader) (err error) { - fileSum += file.Size - entryCount += r.Properties.NumEntries - keySum += r.Properties.RawKeySize - valSum += r.Properties.RawValueSize - return nil - }) + for file = iter.First(); file != nil; file = iter.Next() { + var err error + if file.Virtual { + err = d.tableCache.withVirtualReader(file.VirtualMeta(), addVirtualTableStats) + } else { + err = d.tableCache.withReader(file.PhysicalMeta(), addPhysicalTableStats) + } if err != nil { return 0, 0, err } @@ -409,24 +472,25 @@ func (d *DB) averageEntrySizeBeneath( if entryCount == 0 { return 0, 0, nil } - // RawKeySize and RawValueSize are uncompressed totals. Scale them - // according to the data size to account for compression, index blocks and - // metadata overhead. Eg: + // RawKeySize and RawValueSize are uncompressed totals. We'll need to scale + // the value sum according to the data size to account for compression, + // index blocks and metadata overhead. Eg: // - // Compression rate × Average uncompressed key size + // Compression rate × Average uncompressed value size // // ↓ // - // FileSize RawKeySize - // ----------------------- × ---------- + // FileSize RawValueSize + // ----------------------- × ------------ // RawKeySize+RawValueSize NumEntries // - // We refactor the calculation to avoid error from rounding/truncation. - totalSizePerEntry := fileSum / entryCount - uncompressedSum := keySum + valSum - avgKeySize = keySum * totalSizePerEntry / uncompressedSum - avgValueSize = valSum * totalSizePerEntry / uncompressedSum - return avgKeySize, avgValueSize, err + // We return the average logical value size plus the compression ratio, + // leaving the scaling to the caller. This allows the caller to perform + // additional compression ratio scaling if necessary. + uncompressedSum := float64(keySum + valSum) + compressionRatio = float64(fileSum) / uncompressedSum + avgValueLogicalSize = (float64(valSum) / float64(entryCount)) + return avgValueLogicalSize, compressionRatio, nil } func (d *DB) estimateReclaimedSizeBeneath( @@ -502,10 +566,21 @@ func (d *DB) estimateReclaimedSizeBeneath( continue } var size uint64 - err := d.tableCache.withReader(file, func(r *sstable.Reader) (err error) { - size, err = r.EstimateDiskUsage(start, end) - return err - }) + var err error + if file.Virtual { + err = d.tableCache.withVirtualReader( + file.VirtualMeta(), func(r sstable.VirtualReader) (err error) { + size, err = r.EstimateDiskUsage(start, end) + return err + }) + } else { + err = d.tableCache.withReader( + file.PhysicalMeta(), func(r *sstable.Reader) (err error) { + size, err = r.EstimateDiskUsage(start, end) + return err + }) + } + if err != nil { return 0, hintSeqNum, err } @@ -516,7 +591,7 @@ func (d *DB) estimateReclaimedSizeBeneath( return estimate, hintSeqNum, nil } -func maybeSetStatsFromProperties(meta *fileMetadata, props *sstable.Properties) bool { +func maybeSetStatsFromProperties(meta physicalMeta, props *sstable.Properties) bool { // If a table contains range deletions or range key deletions, we defer the // stats collection. There are two main reasons for this: // @@ -532,12 +607,12 @@ func maybeSetStatsFromProperties(meta *fileMetadata, props *sstable.Properties) return false } - // If a table is more than 10% point deletions, don't calculate the - // PointDeletionsBytesEstimate statistic using our limited knowledge. The - // table stats collector can populate the stats and calculate an average - // of value size of all the tables beneath the table in the LSM, which - // will be more accurate. - if props.NumDeletions > props.NumEntries/10 { + // If a table is more than 10% point deletions without user-provided size + // estimates, don't calculate the PointDeletionsBytesEstimate statistic + // using our limited knowledge. The table stats collector can populate the + // stats and calculate an average of value size of all the tables beneath + // the table in the LSM, which will be more accurate. + if unsizedDels := (props.NumDeletions - props.NumSizedDeletions); unsizedDels > props.NumEntries/10 { return false } @@ -547,8 +622,9 @@ func maybeSetStatsFromProperties(meta *fileMetadata, props *sstable.Properties) // doesn't require any additional IO and since the number of point // deletions in the file is low, the error introduced by this crude // estimate is expected to be small. - avgKeySize, avgValSize := estimateEntrySizes(meta.Size, props) - pointEstimate = pointDeletionsBytesEstimate(props, avgKeySize, avgValSize) + commonProps := &props.CommonProperties + avgValSize, compressionRatio := estimatePhysicalSizes(meta.Size, commonProps) + pointEstimate = pointDeletionsBytesEstimate(meta.Size, commonProps, avgValSize, compressionRatio) } meta.Stats.NumEntries = props.NumEntries @@ -561,47 +637,112 @@ func maybeSetStatsFromProperties(meta *fileMetadata, props *sstable.Properties) return true } -func pointDeletionsBytesEstimate(props *sstable.Properties, avgKeySize, avgValSize uint64) uint64 { +func pointDeletionsBytesEstimate( + fileSize uint64, props *sstable.CommonProperties, avgValLogicalSize, compressionRatio float64, +) (estimate uint64) { if props.NumEntries == 0 { return 0 } - // Estimate the potential space to reclaim using the table's own - // properties. There may or may not be keys covered by any individual - // point tombstone. If not, compacting the point tombstone into L6 will at - // least allow us to drop the point deletion key and will reclaim the key - // bytes. If there are covered key(s), we also get to drop key and value - // bytes for each covered key. + numPointDels := props.NumPointDeletions() + if numPointDels == 0 { + return 0 + } + // Estimate the potential space to reclaim using the table's own properties. + // There may or may not be keys covered by any individual point tombstone. + // If not, compacting the point tombstone into L6 will at least allow us to + // drop the point deletion key and will reclaim the tombstone's key bytes. + // If there are covered key(s), we also get to drop key and value bytes for + // each covered key. // - // We estimate assuming that each point tombstone on average covers 1 key. + // Some point tombstones (DELSIZEDs) carry a user-provided estimate of the + // uncompressed size of entries that will be elided by fully compacting the + // tombstone. For these tombstones, there's no guesswork—we use the + // RawPointTombstoneValueSizeHint property which is the sum of all these + // tombstones' encoded values. + // + // For un-sized point tombstones (DELs), we estimate assuming that each + // point tombstone on average covers 1 key and using average value sizes. // This is almost certainly an overestimate, but that's probably okay // because point tombstones can slow range iterations even when they don't - // cover a key. It may be beneficial in the future to more accurately - // estimate which tombstones cover keys and which do not. - numPointDels := props.NumPointDeletions() - return numPointDels*avgKeySize + numPointDels*(avgKeySize+avgValSize) + // cover a key. + // + // TODO(jackson): This logic doesn't directly incorporate fixed per-key + // overhead (8-byte trailer, plus at least 1 byte encoding the length of the + // key and 1 byte encoding the length of the value). This overhead is + // indirectly incorporated through the compression ratios, but that results + // in the overhead being smeared per key-byte and value-byte, rather than + // per-entry. This per-key fixed overhead can be nontrivial, especially for + // dense swaths of point tombstones. Give some thought as to whether we + // should directly include fixed per-key overhead in the calculations. + + // Below, we calculate the tombstone contributions and the shadowed keys' + // contributions separately. + var tombstonesLogicalSize float64 + var shadowedLogicalSize float64 + + // 1. Calculate the contribution of the tombstone keys themselves. + if props.RawPointTombstoneKeySize > 0 { + tombstonesLogicalSize += float64(props.RawPointTombstoneKeySize) + } else { + // This sstable predates the existence of the RawPointTombstoneKeySize + // property. We can use the average key size within the file itself and + // the count of point deletions to estimate the size. + tombstonesLogicalSize += float64(numPointDels * props.RawKeySize / props.NumEntries) + } + + // 2. Calculate the contribution of the keys shadowed by tombstones. + // + // 2a. First account for keys shadowed by DELSIZED tombstones. THE DELSIZED + // tombstones encode the size of both the key and value of the shadowed KV + // entries. These sizes are aggregated into a sstable property. + shadowedLogicalSize += float64(props.RawPointTombstoneValueSize) + + // 2b. Calculate the contribution of the KV entries shadowed by ordinary DEL + // keys. + numUnsizedDels := numPointDels - props.NumSizedDeletions + { + // The shadowed keys have the same exact user keys as the tombstones + // themselves, so we can use the `tombstonesLogicalSize` we computed + // earlier as an estimate. There's a complication that + // `tombstonesLogicalSize` may include DELSIZED keys we already + // accounted for. + shadowedLogicalSize += float64(tombstonesLogicalSize) / float64(numPointDels) * float64(numUnsizedDels) + + // Calculate the contribution of the deleted values. The caller has + // already computed an average logical size (possibly computed across + // many sstables). + shadowedLogicalSize += float64(numUnsizedDels) * avgValLogicalSize + } + + // Scale both tombstone and shadowed totals by logical:physical ratios to + // account for compression, metadata overhead, etc. + // + // Physical FileSize + // ----------- = ----------------------- + // Logical RawKeySize+RawValueSize + // + return uint64((tombstonesLogicalSize + shadowedLogicalSize) * compressionRatio) } -func estimateEntrySizes( - fileSize uint64, props *sstable.Properties, -) (avgKeySize, avgValSize uint64) { - // RawKeySize and RawValueSize are uncompressed totals. Scale them - // according to the data size to account for compression, index blocks and - // metadata overhead. Eg: +func estimatePhysicalSizes( + fileSize uint64, props *sstable.CommonProperties, +) (avgValLogicalSize, compressionRatio float64) { + // RawKeySize and RawValueSize are uncompressed totals. Scale according to + // the data size to account for compression, index blocks and metadata + // overhead. Eg: // - // Compression rate × Average uncompressed key size + // Compression rate × Average uncompressed value size // // ↓ // - // FileSize RawKeySize + // FileSize RawValSize // ----------------------- × ---------- // RawKeySize+RawValueSize NumEntries // - // We refactor the calculation to avoid error from rounding/truncation. - fileSizePerEntry := fileSize / props.NumEntries uncompressedSum := props.RawKeySize + props.RawValueSize - avgKeySize = props.RawKeySize * fileSizePerEntry / uncompressedSum - avgValSize = props.RawValueSize * fileSizePerEntry / uncompressedSum - return avgKeySize, avgValSize + compressionRatio = float64(fileSize) / float64(uncompressedSum) + avgValLogicalSize = (float64(props.RawValueSize) / float64(props.NumEntries)) + return avgValLogicalSize, compressionRatio } // newCombinedDeletionKeyspanIter returns a keyspan.FragmentIterator that @@ -657,7 +798,7 @@ func estimateEntrySizes( // corresponding to the largest and smallest sequence numbers encountered across // the range deletes and range keys deletes that comprised the merged spans. func newCombinedDeletionKeyspanIter( - comparer *base.Comparer, r *sstable.Reader, m *fileMetadata, + comparer *base.Comparer, cr sstable.CommonReader, m *fileMetadata, ) (keyspan.FragmentIterator, error) { // The range del iter and range key iter are each wrapped in their own // defragmenting iter. For each iter, abutting spans can always be merged. @@ -719,7 +860,7 @@ func newCombinedDeletionKeyspanIter( }) mIter.Init(comparer.Compare, transform, new(keyspan.MergingBuffers)) - iter, err := r.NewRawRangeDelIter() + iter, err := cr.NewRawRangeDelIter() if err != nil { return nil, err } @@ -730,12 +871,13 @@ func newCombinedDeletionKeyspanIter( // Truncate tombstones to the containing file's bounds if necessary. // See docs/range_deletions.md for why this is necessary. iter = keyspan.Truncate( - comparer.Compare, iter, m.Smallest.UserKey, m.Largest.UserKey, nil, nil, + comparer.Compare, iter, m.Smallest.UserKey, m.Largest.UserKey, + nil, nil, false, /* panicOnUpperTruncate */ ) mIter.AddLevel(iter) } - iter, err = r.NewRawRangeKeyIter() + iter, err = cr.NewRawRangeKeyIter() if err != nil { return nil, err } @@ -752,7 +894,7 @@ func newCombinedDeletionKeyspanIter( out.Keys = append(out.Keys, k) } return len(out.Keys) > 0 - }) + }, comparer.Compare) dIter := &keyspan.DefragmentingIter{} dIter.Init(comparer, iter, equal, reducer, new(keyspan.DefragmentingBuffers)) iter = dIter @@ -785,7 +927,7 @@ func (a rangeKeySetsAnnotator) Accumulate( ) (v interface{}, cacheOK bool) { vptr := dst.(*uint64) *vptr = *vptr + f.Stats.NumRangeKeySets - return vptr, f.StatsValidLocked() + return vptr, f.StatsValid() } func (a rangeKeySetsAnnotator) Merge(src interface{}, dst interface{}) interface{} { @@ -832,7 +974,7 @@ func (a tombstonesAnnotator) Accumulate( ) (v interface{}, cacheOK bool) { vptr := dst.(*uint64) *vptr = *vptr + f.Stats.NumDeletions - return vptr, f.StatsValidLocked() + return vptr, f.StatsValid() } func (a tombstonesAnnotator) Merge(src interface{}, dst interface{}) interface{} { @@ -879,7 +1021,7 @@ func (a valueBlocksSizeAnnotator) Accumulate( ) (v interface{}, cacheOK bool) { vptr := dst.(*uint64) *vptr = *vptr + f.Stats.ValueBlocksSize - return vptr, f.StatsValidLocked() + return vptr, f.StatsValid() } func (a valueBlocksSizeAnnotator) Merge(src interface{}, dst interface{}) interface{} { diff --git a/vendor/github.com/cockroachdb/pebble/version_set.go b/vendor/github.com/cockroachdb/pebble/version_set.go index 216f211..5f4c677 100644 --- a/vendor/github.com/cockroachdb/pebble/version_set.go +++ b/vendor/github.com/cockroachdb/pebble/version_set.go @@ -8,7 +8,6 @@ import ( "bytes" "fmt" "io" - "math" "sync" "sync/atomic" @@ -30,6 +29,9 @@ const manifestMarkerName = `manifest` type bulkVersionEdit = manifest.BulkVersionEdit type deletedFileEntry = manifest.DeletedFileEntry type fileMetadata = manifest.FileMetadata +type physicalMeta = manifest.PhysicalFileMeta +type virtualMeta = manifest.VirtualFileMeta +type fileBacking = manifest.FileBacking type newFileEntry = manifest.NewFileEntry type version = manifest.Version type versionEdit = manifest.VersionEdit @@ -41,36 +43,28 @@ type versionList = manifest.VersionList // like it sounds: a delta from the previous version. Version edits are logged // to the MANIFEST file, which is replayed at startup. type versionSet struct { - // WARNING: The following struct `atomic` contains fields are accessed atomically. - // - // Go allocations are guaranteed to be 64-bit aligned which we take advantage - // of by placing the 64-bit fields which we access atomically at the beginning - // of the versionSet struct. - // For more information, see https://golang.org/pkg/sync/atomic/#pkg-note-BUG. - atomic struct { - logSeqNum uint64 // next seqNum to use for WAL writes + // Next seqNum to use for WAL writes. + logSeqNum atomic.Uint64 - // The upper bound on sequence numbers that have been assigned so far. - // A suffix of these sequence numbers may not have been written to a - // WAL. Both logSeqNum and visibleSeqNum are atomically updated by the - // commitPipeline. - visibleSeqNum uint64 // visible seqNum (<= logSeqNum) + // The upper bound on sequence numbers that have been assigned so far. A + // suffix of these sequence numbers may not have been written to a WAL. Both + // logSeqNum and visibleSeqNum are atomically updated by the commitPipeline. + // visibleSeqNum is <= logSeqNum. + visibleSeqNum atomic.Uint64 - // Number of bytes present in sstables being written by in-progress - // compactions. This value will be zero if there are no in-progress - // compactions. Updated and read atomically. - atomicInProgressBytes int64 - } + // Number of bytes present in sstables being written by in-progress + // compactions. This value will be zero if there are no in-progress + // compactions. Updated and read atomically. + atomicInProgressBytes atomic.Int64 // Immutable fields. dirname string // Set to DB.mu. - mu *sync.Mutex - opts *Options - fs vfs.FS - cmp Compare - cmpName string - diskAvailBytes func() uint64 + mu *sync.Mutex + opts *Options + fs vfs.FS + cmp Compare + cmpName string // Dynamic base level allows the dynamic base level computation to be // disabled. Used by tests which want to create specific LSM structures. dynamicBaseLevel bool @@ -83,14 +77,30 @@ type versionSet struct { // A pointer to versionSet.addObsoleteLocked. Avoids allocating a new closure // on the creation of every version. - obsoleteFn func(obsolete []*manifest.FileMetadata) - obsoleteTables []*manifest.FileMetadata + obsoleteFn func(obsolete []*fileBacking) + obsoleteTables []fileInfo obsoleteManifests []fileInfo obsoleteOptions []fileInfo // Zombie tables which have been removed from the current version but are // still referenced by an inuse iterator. - zombieTables map[FileNum]uint64 // filenum -> size + zombieTables map[base.DiskFileNum]uint64 // filenum -> size + + // backingState is protected by the versionSet.logLock. It's populated + // during Open in versionSet.load, but it's not used concurrently during + // load. + backingState struct { + // fileBackingMap is a map for the FileBacking which is supporting virtual + // sstables in the latest version. Once the file backing is backing no + // virtual sstables in the latest version, it is removed from this map and + // the corresponding state is added to the zombieTables map. Note that we + // don't keep track of file backing which supports a virtual sstable + // which is not in the latest version. + fileBackingMap map[base.DiskFileNum]*fileBacking + // fileBackingSize is the sum of the sizes of the fileBackings in the + // fileBackingMap. + fileBackingSize uint64 + } // minUnflushedLogNum is the smallest WAL log file number corresponding to // mutations that have not been flushed to an sstable. @@ -104,9 +114,10 @@ type versionSet struct { manifestFileNum FileNum manifestMarker *atomicfs.Marker - manifestFile vfs.File - manifest *record.Writer - setCurrent func(FileNum) error + manifestFile vfs.File + manifest *record.Writer + setCurrent func(FileNum) error + getFormatMajorVersion func() FormatMajorVersion writing bool writerCond sync.Cond @@ -119,6 +130,7 @@ func (vs *versionSet) init( opts *Options, marker *atomicfs.Marker, setCurrent func(FileNum) error, + getFMV func() FormatMajorVersion, mu *sync.Mutex, ) { vs.dirname = dirname @@ -131,13 +143,13 @@ func (vs *versionSet) init( vs.dynamicBaseLevel = true vs.versions.Init(mu) vs.obsoleteFn = vs.addObsoleteLocked - vs.zombieTables = make(map[FileNum]uint64) + vs.zombieTables = make(map[base.DiskFileNum]uint64) + vs.backingState.fileBackingMap = make(map[base.DiskFileNum]*fileBacking) + vs.backingState.fileBackingSize = 0 vs.nextFileNum = 1 vs.manifestMarker = marker vs.setCurrent = setCurrent - if vs.diskAvailBytes == nil { - vs.diskAvailBytes = func() uint64 { return math.MaxUint64 } - } + vs.getFormatMajorVersion = getFMV } // create creates a version set for a fresh DB. @@ -147,15 +159,15 @@ func (vs *versionSet) create( opts *Options, marker *atomicfs.Marker, setCurrent func(FileNum) error, + getFormatMajorVersion func() FormatMajorVersion, mu *sync.Mutex, ) error { - vs.init(dirname, opts, marker, setCurrent, mu) + vs.init(dirname, opts, marker, setCurrent, getFormatMajorVersion, mu) newVersion := &version{} vs.append(newVersion) var err error - vs.picker = newCompactionPicker(newVersion, vs.opts, nil, vs.metrics.levelSizes(), vs.diskAvailBytes) - + vs.picker = newCompactionPicker(newVersion, vs.opts, nil) // Note that a "snapshot" version edit is written to the manifest when it is // created. vs.manifestFileNum = vs.getNextFileNum() @@ -179,7 +191,7 @@ func (vs *versionSet) create( vs.opts.EventListener.ManifestCreated(ManifestCreateInfo{ JobID: jobID, - Path: base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, vs.manifestFileNum), + Path: base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, vs.manifestFileNum.DiskFileNum()), FileNum: vs.manifestFileNum, Err: err, }) @@ -196,12 +208,13 @@ func (vs *versionSet) load( manifestFileNum FileNum, marker *atomicfs.Marker, setCurrent func(FileNum) error, + getFormatMajorVersion func() FormatMajorVersion, mu *sync.Mutex, ) error { - vs.init(dirname, opts, marker, setCurrent, mu) + vs.init(dirname, opts, marker, setCurrent, getFormatMajorVersion, mu) vs.manifestFileNum = manifestFileNum - manifestPath := base.MakeFilepath(opts.FS, dirname, fileTypeManifest, vs.manifestFileNum) + manifestPath := base.MakeFilepath(opts.FS, dirname, fileTypeManifest, vs.manifestFileNum.DiskFileNum()) manifestFilename := opts.FS.PathBase(manifestPath) // Read the versionEdits in the manifest file. @@ -257,7 +270,14 @@ func (vs *versionSet) load( // (assuming no WALs contain higher sequence numbers than the // manifest's LastSeqNum). Increment LastSeqNum by 1 to get the // next sequence number that will be assigned. - vs.atomic.logSeqNum = ve.LastSeqNum + 1 + // + // If LastSeqNum is less than SeqNumStart, increase it to at least + // SeqNumStart to leave ample room for reserved sequence numbers. + if ve.LastSeqNum+1 < base.SeqNumStart { + vs.logSeqNum.Store(base.SeqNumStart) + } else { + vs.logSeqNum.Store(ve.LastSeqNum + 1) + } } } // We have already set vs.nextFileNum = 2 at the beginning of the @@ -277,7 +297,21 @@ func (vs *versionSet) load( } vs.markFileNumUsed(vs.minUnflushedLogNum) - newVersion, _, err := bve.Apply(nil, vs.cmp, opts.Comparer.FormatKey, opts.FlushSplitBytes, opts.Experimental.ReadCompactionRate) + // Populate the fileBackingMap and the FileBacking for virtual sstables since + // we have finished version edit accumulation. + for _, s := range bve.AddedFileBacking { + vs.addFileBacking(s) + } + + for _, fileNum := range bve.RemovedFileBacking { + vs.removeFileBacking(fileNum) + } + + newVersion, err := bve.Apply( + nil, vs.cmp, opts.Comparer.FormatKey, opts.FlushSplitBytes, + opts.Experimental.ReadCompactionRate, nil, /* zombies */ + getFormatMajorVersion().orderingInvariants(), + ) if err != nil { return err } @@ -291,7 +325,7 @@ func (vs *versionSet) load( l.Size = int64(files.SizeSum()) } - vs.picker = newCompactionPicker(newVersion, vs.opts, nil, vs.metrics.levelSizes(), vs.diskAvailBytes) + vs.picker = newCompactionPicker(newVersion, vs.opts, nil) return nil } @@ -334,6 +368,26 @@ func (vs *versionSet) logUnlock() { vs.writerCond.Signal() } +// Only call if the DiskFileNum doesn't exist in the fileBackingMap. +func (vs *versionSet) addFileBacking(backing *manifest.FileBacking) { + _, ok := vs.backingState.fileBackingMap[backing.DiskFileNum] + if ok { + panic("pebble: trying to add an existing file backing") + } + vs.backingState.fileBackingMap[backing.DiskFileNum] = backing + vs.backingState.fileBackingSize += backing.Size +} + +// Only call if the the DiskFileNum exists in the fileBackingMap. +func (vs *versionSet) removeFileBacking(dfn base.DiskFileNum) { + backing, ok := vs.backingState.fileBackingMap[dfn] + if !ok { + panic("pebble: trying to remove an unknown file backing") + } + delete(vs.backingState.fileBackingMap, dfn) + vs.backingState.fileBackingSize -= backing.Size +} + // logAndApply logs the version edit to the manifest, applies the version edit // to the current version, and installs the new version. // @@ -382,7 +436,7 @@ func (vs *versionSet) logAndApply( // in an unflushed memtable. logSeqNum is the _next_ sequence number that // will be assigned, so subtract that by 1 to get the upper bound on the // last assigned sequence number. - logSeqNum := atomic.LoadUint64(&vs.atomic.logSeqNum) + logSeqNum := vs.logSeqNum.Load() ve.LastSeqNum = logSeqNum - 1 if logSeqNum == 0 { // logSeqNum is initialized to 1 in Open() if there are no previous WAL @@ -391,6 +445,8 @@ func (vs *versionSet) logAndApply( } currentVersion := vs.currentVersion() + fmv := vs.getFormatMajorVersion() + orderingInvariants := fmv.orderingInvariants() var newVersion *version // Generate a new manifest if we don't currently have one, or forceRotation @@ -455,18 +511,21 @@ func (vs *versionSet) logAndApply( minUnflushedLogNum := vs.minUnflushedLogNum nextFileNum := vs.nextFileNum - var zombies map[FileNum]uint64 + var zombies map[base.DiskFileNum]uint64 if err := func() error { vs.mu.Unlock() defer vs.mu.Lock() - var bve bulkVersionEdit - if err := bve.Accumulate(ve); err != nil { - return err - } - var err error - newVersion, zombies, err = bve.Apply(currentVersion, vs.cmp, vs.opts.Comparer.FormatKey, vs.opts.FlushSplitBytes, vs.opts.Experimental.ReadCompactionRate) + if vs.getFormatMajorVersion() < FormatVirtualSSTables && len(ve.CreatedBackingTables) > 0 { + return errors.AssertionFailedf("MANIFEST cannot contain virtual sstable records due to format major version") + } + newVersion, zombies, err = manifest.AccumulateIncompleteAndApplySingleVE( + ve, currentVersion, vs.cmp, vs.opts.Comparer.FormatKey, + vs.opts.FlushSplitBytes, vs.opts.Experimental.ReadCompactionRate, + vs.backingState.fileBackingMap, vs.addFileBacking, vs.removeFileBacking, + orderingInvariants, + ) if err != nil { return errors.Wrap(err, "MANIFEST apply failed") } @@ -475,7 +534,7 @@ func (vs *versionSet) logAndApply( if err := vs.createManifest(vs.dirname, newManifestFileNum, minUnflushedLogNum, nextFileNum); err != nil { vs.opts.EventListener.ManifestCreated(ManifestCreateInfo{ JobID: jobID, - Path: base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, newManifestFileNum), + Path: base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, newManifestFileNum.DiskFileNum()), FileNum: newManifestFileNum, Err: err, }) @@ -487,7 +546,8 @@ func (vs *versionSet) logAndApply( if err != nil { return errors.Wrap(err, "MANIFEST next record write failed") } - // NB: Any error from this point on is considered fatal as we don't now if + + // NB: Any error from this point on is considered fatal as we don't know if // the MANIFEST write occurred or not. Trying to determine that is // fraught. Instead we rely on the standard recovery mechanism run when a // database is open. In particular, that mechanism generates a new MANIFEST @@ -508,7 +568,7 @@ func (vs *versionSet) logAndApply( } vs.opts.EventListener.ManifestCreated(ManifestCreateInfo{ JobID: jobID, - Path: base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, newManifestFileNum), + Path: base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, newManifestFileNum.DiskFileNum()), FileNum: newManifestFileNum, }) } @@ -549,7 +609,7 @@ func (vs *versionSet) logAndApply( if newManifestFileNum != 0 { if vs.manifestFileNum != 0 { vs.obsoleteManifests = append(vs.obsoleteManifests, fileInfo{ - fileNum: vs.manifestFileNum, + fileNum: vs.manifestFileNum.DiskFileNum(), fileSize: prevManifestFileSize, }) } @@ -561,30 +621,46 @@ func (vs *versionSet) logAndApply( } for i := range vs.metrics.Levels { l := &vs.metrics.Levels[i] + l.NumFiles = int64(newVersion.Levels[i].Len()) + l.NumVirtualFiles = newVersion.Levels[i].NumVirtual + l.VirtualSize = newVersion.Levels[i].VirtualSize + l.Size = int64(newVersion.Levels[i].Size()) + l.Sublevels = 0 if l.NumFiles > 0 { l.Sublevels = 1 } if invariants.Enabled { - if count := int64(newVersion.Levels[i].Len()); l.NumFiles != count { - vs.opts.Logger.Fatalf("versionSet metrics L%d NumFiles = %d, actual count = %d", i, l.NumFiles, count) - } levelFiles := newVersion.Levels[i].Slice() if size := int64(levelFiles.SizeSum()); l.Size != size { vs.opts.Logger.Fatalf("versionSet metrics L%d Size = %d, actual size = %d", i, l.Size, size) } + if nVirtual := levelFiles.NumVirtual(); nVirtual != l.NumVirtualFiles { + vs.opts.Logger.Fatalf( + "versionSet metrics L%d NumVirtual = %d, actual NumVirtual = %d", + i, l.NumVirtualFiles, nVirtual, + ) + } + if vSize := levelFiles.VirtualSizeSum(); vSize != l.VirtualSize { + vs.opts.Logger.Fatalf( + "versionSet metrics L%d Virtual size = %d, actual size = %d", + i, l.VirtualSize, vSize, + ) + } } } vs.metrics.Levels[0].Sublevels = int32(len(newVersion.L0SublevelFiles)) - vs.picker = newCompactionPicker(newVersion, vs.opts, inProgress, vs.metrics.levelSizes(), vs.diskAvailBytes) + vs.picker = newCompactionPicker(newVersion, vs.opts, inProgress) if !vs.dynamicBaseLevel { vs.picker.forceBaseLevel1() } return nil } -func (vs *versionSet) incrementCompactions(kind compactionKind, extraLevels []*compactionLevel) { +func (vs *versionSet) incrementCompactions( + kind compactionKind, extraLevels []*compactionLevel, pickerMetrics compactionPickerMetrics, +) { switch kind { case compactionKindDefault: vs.metrics.Compact.Count++ @@ -619,7 +695,7 @@ func (vs *versionSet) incrementCompactions(kind compactionKind, extraLevels []*c } func (vs *versionSet) incrementCompactionBytes(numBytes int64) { - atomic.AddInt64(&vs.atomic.atomicInProgressBytes, numBytes) + vs.atomicInProgressBytes.Add(numBytes) } // createManifest creates a manifest file that contains a snapshot of vs. @@ -627,7 +703,7 @@ func (vs *versionSet) createManifest( dirname string, fileNum, minUnflushedLogNum, nextFileNum FileNum, ) (err error) { var ( - filename = base.MakeFilepath(vs.fs, dirname, fileTypeManifest, fileNum) + filename = base.MakeFilepath(vs.fs, dirname, fileTypeManifest, fileNum.DiskFileNum()) manifestFile vfs.File manifest *record.Writer ) @@ -651,6 +727,7 @@ func (vs *versionSet) createManifest( snapshot := versionEdit{ ComparerName: vs.cmpName, } + dedup := make(map[base.DiskFileNum]struct{}) for level, levelMetadata := range vs.currentVersion().Levels { iter := levelMetadata.Iter() for meta := iter.First(); meta != nil; meta = iter.Next() { @@ -658,6 +735,13 @@ func (vs *versionSet) createManifest( Level: level, Meta: meta, }) + if _, ok := dedup[meta.FileBacking.DiskFileNum]; meta.Virtual && !ok { + dedup[meta.FileBacking.DiskFileNum] = struct{}{} + snapshot.CreatedBackingTables = append( + snapshot.CreatedBackingTables, + meta.FileBacking, + ) + } } } @@ -722,13 +806,13 @@ func (vs *versionSet) currentVersion() *version { return vs.versions.Back() } -func (vs *versionSet) addLiveFileNums(m map[FileNum]struct{}) { +func (vs *versionSet) addLiveFileNums(m map[base.DiskFileNum]struct{}) { current := vs.currentVersion() for v := vs.versions.Front(); true; v = v.Next() { for _, lm := range v.Levels { iter := lm.Iter() for f := iter.First(); f != nil; f = iter.Next() { - m[f.FileNum] = struct{}{} + m[f.FileBacking.DiskFileNum] = struct{}{} } } if v == current { @@ -737,27 +821,49 @@ func (vs *versionSet) addLiveFileNums(m map[FileNum]struct{}) { } } +// addObsoleteLocked will add the fileInfo associated with obsolete backing +// sstables to the obsolete tables list. +// +// The file backings in the obsolete list must not appear more than once. +// // DB.mu must be held when addObsoleteLocked is called. -func (vs *versionSet) addObsoleteLocked(obsolete []*manifest.FileMetadata) { +func (vs *versionSet) addObsoleteLocked(obsolete []*fileBacking) { if len(obsolete) == 0 { return } - for _, fileMeta := range obsolete { + obsoleteFileInfo := make([]fileInfo, len(obsolete)) + for i, bs := range obsolete { + obsoleteFileInfo[i].fileNum = bs.DiskFileNum + obsoleteFileInfo[i].fileSize = bs.Size + } + + if invariants.Enabled { + dedup := make(map[base.DiskFileNum]struct{}) + for _, fi := range obsoleteFileInfo { + dedup[fi.fileNum] = struct{}{} + } + if len(dedup) != len(obsoleteFileInfo) { + panic("pebble: duplicate FileBacking present in obsolete list") + } + } + + for _, fi := range obsoleteFileInfo { // Note that the obsolete tables are no longer zombie by the definition of // zombie, but we leave them in the zombie tables map until they are // deleted from disk. - if _, ok := vs.zombieTables[fileMeta.FileNum]; !ok { - vs.opts.Logger.Fatalf("MANIFEST obsolete table %s not marked as zombie", fileMeta.FileNum) + if _, ok := vs.zombieTables[fi.fileNum]; !ok { + vs.opts.Logger.Fatalf("MANIFEST obsolete table %s not marked as zombie", fi.fileNum) } } - vs.obsoleteTables = append(vs.obsoleteTables, obsolete...) + + vs.obsoleteTables = append(vs.obsoleteTables, obsoleteFileInfo...) vs.updateObsoleteTableMetricsLocked() } // addObsolete will acquire DB.mu, so DB.mu must not be held when this is // called. -func (vs *versionSet) addObsolete(obsolete []*manifest.FileMetadata) { +func (vs *versionSet) addObsolete(obsolete []*fileBacking) { vs.mu.Lock() defer vs.mu.Unlock() vs.addObsoleteLocked(obsolete) @@ -766,8 +872,8 @@ func (vs *versionSet) addObsolete(obsolete []*manifest.FileMetadata) { func (vs *versionSet) updateObsoleteTableMetricsLocked() { vs.metrics.Table.ObsoleteCount = int64(len(vs.obsoleteTables)) vs.metrics.Table.ObsoleteSize = 0 - for _, fileMeta := range vs.obsoleteTables { - vs.metrics.Table.ObsoleteSize += fileMeta.Size + for _, fi := range vs.obsoleteTables { + vs.metrics.Table.ObsoleteSize += fi.fileSize } } @@ -779,7 +885,7 @@ func setCurrentFunc( // the CURRENT file to signal which MANIFEST is current. Ignore // the filename read during LocateMarker. return func(manifestFileNum FileNum) error { - if err := setCurrentFile(dirname, fs, manifestFileNum); err != nil { + if err := setCurrentFile(dirname, fs, manifestFileNum.DiskFileNum()); err != nil { return err } if err := dir.Sync(); err != nil { @@ -797,13 +903,13 @@ func setCurrentFunc( func setCurrentFuncMarker(marker *atomicfs.Marker, fs vfs.FS, dirname string) func(FileNum) error { return func(manifestFileNum FileNum) error { - return marker.Move(base.MakeFilename(fileTypeManifest, manifestFileNum)) + return marker.Move(base.MakeFilename(fileTypeManifest, manifestFileNum.DiskFileNum())) } } func findCurrentManifest( vers FormatMajorVersion, fs vfs.FS, dirname string, -) (marker *atomicfs.Marker, manifestNum FileNum, exists bool, err error) { +) (marker *atomicfs.Marker, manifestNum base.DiskFileNum, exists bool, err error) { // NB: We always locate the manifest marker, even if we might not // actually use it (because we're opening the database at an earlier // format major version that uses the CURRENT file). Locating a @@ -811,7 +917,7 @@ func findCurrentManifest( var filename string marker, filename, err = atomicfs.LocateMarker(fs, dirname, manifestMarkerName) if err != nil { - return nil, 0, false, err + return nil, base.FileNum(0).DiskFileNum(), false, err } if vers < formatVersionedManifestMarker { @@ -821,9 +927,9 @@ func findCurrentManifest( manifestNum, err = readCurrentFile(fs, dirname) if oserror.IsNotExist(err) { - return marker, 0, false, nil + return marker, base.FileNum(0).DiskFileNum(), false, nil } else if err != nil { - return marker, 0, false, err + return marker, base.FileNum(0).DiskFileNum(), false, err } return marker, manifestNum, true, nil } @@ -834,48 +940,48 @@ func findCurrentManifest( if filename == "" { // The marker hasn't been set yet. This database doesn't exist. - return marker, 0, false, nil + return marker, base.FileNum(0).DiskFileNum(), false, nil } var ok bool _, manifestNum, ok = base.ParseFilename(fs, filename) if !ok { - return marker, 0, false, base.CorruptionErrorf("pebble: MANIFEST name %q is malformed", errors.Safe(filename)) + return marker, base.FileNum(0).DiskFileNum(), false, base.CorruptionErrorf("pebble: MANIFEST name %q is malformed", errors.Safe(filename)) } return marker, manifestNum, true, nil } -func readCurrentFile(fs vfs.FS, dirname string) (FileNum, error) { +func readCurrentFile(fs vfs.FS, dirname string) (base.DiskFileNum, error) { // Read the CURRENT file to find the current manifest file. - current, err := fs.Open(base.MakeFilepath(fs, dirname, fileTypeCurrent, 0)) + current, err := fs.Open(base.MakeFilepath(fs, dirname, fileTypeCurrent, base.FileNum(0).DiskFileNum())) if err != nil { - return 0, errors.Wrapf(err, "pebble: could not open CURRENT file for DB %q", dirname) + return base.FileNum(0).DiskFileNum(), errors.Wrapf(err, "pebble: could not open CURRENT file for DB %q", dirname) } defer current.Close() stat, err := current.Stat() if err != nil { - return 0, err + return base.FileNum(0).DiskFileNum(), err } n := stat.Size() if n == 0 { - return 0, errors.Errorf("pebble: CURRENT file for DB %q is empty", dirname) + return base.FileNum(0).DiskFileNum(), errors.Errorf("pebble: CURRENT file for DB %q is empty", dirname) } if n > 4096 { - return 0, errors.Errorf("pebble: CURRENT file for DB %q is too large", dirname) + return base.FileNum(0).DiskFileNum(), errors.Errorf("pebble: CURRENT file for DB %q is too large", dirname) } b := make([]byte, n) _, err = current.ReadAt(b, 0) if err != nil { - return 0, err + return base.FileNum(0).DiskFileNum(), err } if b[n-1] != '\n' { - return 0, base.CorruptionErrorf("pebble: CURRENT file for DB %q is malformed", dirname) + return base.FileNum(0).DiskFileNum(), base.CorruptionErrorf("pebble: CURRENT file for DB %q is malformed", dirname) } b = bytes.TrimSpace(b) _, manifestFileNum, ok := base.ParseFilename(fs, string(b)) if !ok { - return 0, base.CorruptionErrorf("pebble: MANIFEST name %q is malformed", errors.Safe(b)) + return base.FileNum(0).DiskFileNum(), base.CorruptionErrorf("pebble: MANIFEST name %q is malformed", errors.Safe(b)) } return manifestFileNum, nil } diff --git a/vendor/github.com/cockroachdb/pebble/vfs/default_linux_noarm.go b/vendor/github.com/cockroachdb/pebble/vfs/default_linux.go similarity index 98% rename from vendor/github.com/cockroachdb/pebble/vfs/default_linux_noarm.go rename to vendor/github.com/cockroachdb/pebble/vfs/default_linux.go index 2e6c5e6..ec29074 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/default_linux_noarm.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/default_linux.go @@ -2,8 +2,8 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -//go:build linux && !arm -// +build linux,!arm +//go:build linux +// +build linux package vfs diff --git a/vendor/github.com/cockroachdb/pebble/vfs/default_linux_arm.go b/vendor/github.com/cockroachdb/pebble/vfs/default_linux_arm.go deleted file mode 100644 index 338b3c0..0000000 --- a/vendor/github.com/cockroachdb/pebble/vfs/default_linux_arm.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -//go:build linux && arm -// +build linux,arm - -package vfs - -import ( - "os" - "syscall" - - "github.com/cockroachdb/errors" - "golang.org/x/sys/unix" -) - -func wrapOSFileImpl(f *os.File) File { - lf := &linuxOnArmFile{File: f, fd: f.Fd()} - if lf.fd != InvalidFd { - lf.useSyncRange = isSyncRangeSupported(lf.fd) - } - return lf -} - -func (defaultFS) OpenDir(name string) (File, error) { - f, err := os.OpenFile(name, syscall.O_CLOEXEC, 0) - if err != nil { - return nil, errors.WithStack(err) - } - return &linuxOnArmDir{f}, nil -} - -// Assert that linuxOnArmFile and linuxOnArmDir implement vfs.File. -var ( - _ File = (*linuxOnArmDir)(nil) - _ File = (*linuxOnArmFile)(nil) -) - -type linuxOnArmDir struct { - *os.File -} - -func (d *linuxOnArmDir) Prefetch(offset int64, length int64) error { return nil } -func (d *linuxOnArmDir) Preallocate(offset, length int64) error { return nil } -func (d *linuxOnArmDir) SyncData() error { return d.Sync() } -func (d *linuxOnArmDir) SyncTo(offset int64) (fullSync bool, err error) { return false, nil } - -type linuxOnArmFile struct { - *os.File - fd uintptr - useSyncRange bool -} - -func (f *linuxOnArmFile) Prefetch(offset int64, length int64) error { - _, _, err := unix.Syscall(unix.SYS_READAHEAD, uintptr(f.fd), uintptr(offset), uintptr(length)) - return err -} - -func (f *linuxOnArmFile) Preallocate(offset, length int64) error { return nil } - -func (f *linuxOnArmFile) SyncData() error { - // TODO(radu): does arm support unix.Fdatasync? - return f.Sync() -} - -func (f *linuxOnArmFile) SyncTo(int64) (fullSync bool, err error) { - // syscall.SyncFileRange is not defined form arm (see https://github.com/cockroachdb/pebble/issues/171). - if err = f.Sync(); err != nil { - return false, err - } - return true, nil -} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/disk_full.go b/vendor/github.com/cockroachdb/pebble/vfs/disk_full.go index c6542a0..16f6a5d 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/disk_full.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/disk_full.go @@ -39,68 +39,66 @@ func OnDiskFull(fs FS, fn func()) FS { } type enospcFS struct { - inner FS - atomic struct { - // generation is a monotonically increasing number that encodes the - // current state of ENOSPC error handling. Incoming writes are - // organized into generations to provide strong guarantees on when the - // disk full callback is invoked. The callback is invoked once per - // write generation. - // - // Special significance is given to the parity of this generation - // field to optimize incoming writes in the normal state, which only - // need to perform a single atomic load. If generation is odd, an - // ENOSPC error is being actively handled. The generations associated - // with writes are always even. - // - // The lifecycle of a write is: - // - // 1. Atomically load the current generation. - // a. If it's even, this is the write's generation number. - // b. If it's odd, an ENOSPC was recently encountered and the - // corresponding invocation of the disk full callback has not - // yet completed. The write must wait until the callback has - // completed and generation is updated to an even number, which - // becomes the write's generation number. - // 2. Perform the write. If it encounters no error or an error other - // than ENOSPC, the write returns and proceeds no further in this - // lifecycle. - // 3. Handle ENOSPC. If the write encounters ENOSPC, the callback must - // be invoked for the write's generation. The write's goroutine - // acquires the FS's mutex. - // a. If the FS's current generation is still equal to the write's - // generation, the write is the first write of its generation to - // encounter ENOSPC. It increments the FS's current generation - // to an odd number, signifying that an ENOSPC is being handled - // and invokes the callback. - // b. If the FS's current generation has changed, some other write - // from the same generation encountered an ENOSPC first. This - // write waits on the condition variable until the FS's current - // generation is updated indicating that the generation's - // callback invocation has completed. - // 3. Retry the write once. The callback for the write's generation - // has completed, either by this write's goroutine or another's. - // The write may proceed with the expectation that the callback - // remedied the full disk by freeing up disk space and an ENOSPC - // should not be encountered again for at least a few minutes. If - // we do encounter another ENOSPC on the retry, the callback was - // unable to remedy the full disk and another retry won't be - // useful. Any error, including ENOSPC, during the retry is - // returned without further handling. None of the retries invoke - // the callback. - // - // This scheme has a few nice properties: - // * Once the disk-full callback completes, it won't be invoked - // again unless a write that started strictly later encounters an - // ENOSPC. This is convenient if the callback strives to 'fix' the - // full disk, for example, by removing a ballast file. A new - // invocation of the callback guarantees a new problem. - // * Incoming writes block if there's an unhandled ENOSPC. Some - // writes, like WAL or MANIFEST fsyncs, are fatal if they encounter - // an ENOSPC. - generation uint32 - } - mu struct { + inner FS + // generation is a monotonically increasing number that encodes the + // current state of ENOSPC error handling. Incoming writes are + // organized into generations to provide strong guarantees on when the + // disk full callback is invoked. The callback is invoked once per + // write generation. + // + // Special significance is given to the parity of this generation + // field to optimize incoming writes in the normal state, which only + // need to perform a single atomic load. If generation is odd, an + // ENOSPC error is being actively handled. The generations associated + // with writes are always even. + // + // The lifecycle of a write is: + // + // 1. Atomically load the current generation. + // a. If it's even, this is the write's generation number. + // b. If it's odd, an ENOSPC was recently encountered and the + // corresponding invocation of the disk full callback has not + // yet completed. The write must wait until the callback has + // completed and generation is updated to an even number, which + // becomes the write's generation number. + // 2. Perform the write. If it encounters no error or an error other + // than ENOSPC, the write returns and proceeds no further in this + // lifecycle. + // 3. Handle ENOSPC. If the write encounters ENOSPC, the callback must + // be invoked for the write's generation. The write's goroutine + // acquires the FS's mutex. + // a. If the FS's current generation is still equal to the write's + // generation, the write is the first write of its generation to + // encounter ENOSPC. It increments the FS's current generation + // to an odd number, signifying that an ENOSPC is being handled + // and invokes the callback. + // b. If the FS's current generation has changed, some other write + // from the same generation encountered an ENOSPC first. This + // write waits on the condition variable until the FS's current + // generation is updated indicating that the generation's + // callback invocation has completed. + // 3. Retry the write once. The callback for the write's generation + // has completed, either by this write's goroutine or another's. + // The write may proceed with the expectation that the callback + // remedied the full disk by freeing up disk space and an ENOSPC + // should not be encountered again for at least a few minutes. If + // we do encounter another ENOSPC on the retry, the callback was + // unable to remedy the full disk and another retry won't be + // useful. Any error, including ENOSPC, during the retry is + // returned without further handling. None of the retries invoke + // the callback. + // + // This scheme has a few nice properties: + // * Once the disk-full callback completes, it won't be invoked + // again unless a write that started strictly later encounters an + // ENOSPC. This is convenient if the callback strives to 'fix' the + // full disk, for example, by removing a ballast file. A new + // invocation of the callback guarantees a new problem. + // * Incoming writes block if there's an unhandled ENOSPC. Some + // writes, like WAL or MANIFEST fsyncs, are fatal if they encounter + // an ENOSPC. + generation atomic.Uint32 + mu struct { sync.Mutex sync.Cond onDiskFull func() @@ -119,7 +117,7 @@ func (fs *enospcFS) Unwrap() FS { // waitUntilReady blocks until the callback returns. The returned generation // is always even. func (fs *enospcFS) waitUntilReady() uint32 { - gen := atomic.LoadUint32(&fs.atomic.generation) + gen := fs.generation.Load() if gen%2 == 0 { // An even generation indicates that we're not currently handling an // ENOSPC. Allow the write to proceed. @@ -132,10 +130,10 @@ func (fs *enospcFS) waitUntilReady() uint32 { defer fs.mu.Unlock() // Load the generation again with fs.mu locked. - gen = atomic.LoadUint32(&fs.atomic.generation) + gen = fs.generation.Load() for gen%2 == 1 { fs.mu.Wait() - gen = atomic.LoadUint32(&fs.atomic.generation) + gen = fs.generation.Load() } return gen } @@ -144,7 +142,7 @@ func (fs *enospcFS) handleENOSPC(gen uint32) { fs.mu.Lock() defer fs.mu.Unlock() - currentGeneration := atomic.LoadUint32(&fs.atomic.generation) + currentGeneration := fs.generation.Load() // If the current generation is still `gen`, this is the first goroutine // to hit an ENOSPC within this write generation, so this goroutine is @@ -153,7 +151,7 @@ func (fs *enospcFS) handleENOSPC(gen uint32) { // Increment the generation to an odd number, indicating that the FS // is out-of-disk space and incoming writes should pause and wait for // the next generation before continuing. - atomic.StoreUint32(&fs.atomic.generation, gen+1) + fs.generation.Store(gen + 1) func() { // Drop the mutex while we invoke the callback, re-acquiring @@ -165,7 +163,7 @@ func (fs *enospcFS) handleENOSPC(gen uint32) { // Update the current generation again to an even number, indicating // that the callback has completed for the write generation `gen`. - atomic.StoreUint32(&fs.atomic.generation, gen+2) + fs.generation.Store(gen + 2) fs.mu.Broadcast() return } @@ -185,7 +183,7 @@ func (fs *enospcFS) handleENOSPC(gen uint32) { // subsequent generations' callbacks. for currentGeneration == gen+1 { fs.mu.Wait() - currentGeneration = atomic.LoadUint32(&fs.atomic.generation) + currentGeneration = fs.generation.Load() } } @@ -230,6 +228,17 @@ func (fs *enospcFS) Open(name string, opts ...OpenOption) (File, error) { return f, err } +func (fs *enospcFS) OpenReadWrite(name string, opts ...OpenOption) (File, error) { + f, err := fs.inner.OpenReadWrite(name, opts...) + if f != nil { + f = &enospcFile{ + fs: fs, + inner: f, + } + } + return f, err +} + func (fs *enospcFS) OpenDir(name string) (File, error) { f, err := fs.inner.OpenDir(name) if f != nil { @@ -377,6 +386,20 @@ func (f *enospcFile) Write(p []byte) (n int, err error) { return n, err } +func (f *enospcFile) WriteAt(p []byte, ofs int64) (n int, err error) { + gen := f.fs.waitUntilReady() + + n, err = f.inner.WriteAt(p, ofs) + + if err != nil && isENOSPC(err) { + f.fs.handleENOSPC(gen) + var n2 int + n2, err = f.inner.WriteAt(p[n:], ofs+int64(n)) + n += n2 + } + return n, err +} + func (f *enospcFile) Prefetch(offset, length int64) error { return f.inner.Prefetch(offset, length) } diff --git a/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go b/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go index 1c0b9c5..b7d328e 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go @@ -13,7 +13,6 @@ import ( "sync/atomic" "time" - "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/redact" ) @@ -135,7 +134,7 @@ type diskHealthCheckingFile struct { // // NB: this packing scheme is not persisted, and is therefore safe to adjust // across process boundaries. - lastWritePacked uint64 + lastWritePacked atomic.Uint64 createTime time.Time } @@ -174,7 +173,7 @@ func (d *diskHealthCheckingFile) startTicker() { return case <-ticker.C: - packed := atomic.LoadUint64(&d.lastWritePacked) + packed := d.lastWritePacked.Load() if packed == 0 { continue } @@ -219,6 +218,14 @@ func (d *diskHealthCheckingFile) Write(p []byte) (n int, err error) { return n, err } +// Write implements the io.WriterAt interface. +func (d *diskHealthCheckingFile) WriteAt(p []byte, ofs int64) (n int, err error) { + d.timeDiskOp(OpTypeWrite, int64(len(p)), func() { + n, err = d.file.WriteAt(p, ofs) + }) + return n, err +} + // Close implements the io.Closer interface. func (d *diskHealthCheckingFile) Close() error { d.stopTicker() @@ -279,20 +286,12 @@ func (d *diskHealthCheckingFile) timeDiskOp(opType OpType, writeSizeInBytes int6 delta := time.Since(d.createTime) packed := pack(delta, writeSizeInBytes, opType) - if invariants.Enabled { - if !atomic.CompareAndSwapUint64(&d.lastWritePacked, 0, packed) { - panic("concurrent write operations detected on file") - } - } else { - atomic.StoreUint64(&d.lastWritePacked, packed) + if d.lastWritePacked.Swap(packed) != 0 { + panic("concurrent write operations detected on file") } defer func() { - if invariants.Enabled { - if !atomic.CompareAndSwapUint64(&d.lastWritePacked, packed, 0) { - panic("concurrent write operations detected on file") - } - } else { - atomic.StoreUint64(&d.lastWritePacked, 0) + if d.lastWritePacked.Swap(0) != packed { + panic("concurrent write operations detected on file") } }() op() @@ -375,9 +374,17 @@ func (i DiskSlowInfo) String() string { // SafeFormat implements redact.SafeFormatter. func (i DiskSlowInfo) SafeFormat(w redact.SafePrinter, _ rune) { - w.Printf("disk slowness detected: %s on file %s (%d bytes) has been ongoing for %0.1fs", - redact.Safe(i.OpType.String()), redact.Safe(filepath.Base(i.Path)), - redact.Safe(i.WriteSize), redact.Safe(i.Duration.Seconds())) + switch i.OpType { + // Operations for which i.WriteSize is meaningful. + case OpTypeWrite, OpTypeSyncTo, OpTypePreallocate: + w.Printf("disk slowness detected: %s on file %s (%d bytes) has been ongoing for %0.1fs", + redact.Safe(i.OpType.String()), redact.Safe(filepath.Base(i.Path)), + redact.Safe(i.WriteSize), redact.Safe(i.Duration.Seconds())) + default: + w.Printf("disk slowness detected: %s on file %s has been ongoing for %0.1fs", + redact.Safe(i.OpType.String()), redact.Safe(filepath.Base(i.Path)), + redact.Safe(i.Duration.Seconds())) + } } // diskHealthCheckingFS adds disk-health checking facilities to a VFS. @@ -443,7 +450,7 @@ type diskHealthCheckingFS struct { type slot struct { name string opType OpType - startNanos int64 + startNanos atomic.Int64 } // diskHealthCheckingFS implements FS. @@ -510,12 +517,12 @@ func (d *diskHealthCheckingFS) timeFilesystemOp(name string, opType OpType, op f startNanos := time.Now().UnixNano() for i := 0; i < len(d.mu.inflight); i++ { - if atomic.LoadInt64(&d.mu.inflight[i].startNanos) == 0 { + if d.mu.inflight[i].startNanos.Load() == 0 { // This slot is not in use. Claim it. s = d.mu.inflight[i] s.name = name s.opType = opType - atomic.StoreInt64(&s.startNanos, startNanos) + s.startNanos.Store(startNanos) break } } @@ -526,10 +533,10 @@ func (d *diskHealthCheckingFS) timeFilesystemOp(name string, opType OpType, op f // incur an allocation. if s == nil { s = &slot{ - name: name, - opType: opType, - startNanos: startNanos, + name: name, + opType: opType, } + s.startNanos.Store(startNanos) d.mu.inflight = append(d.mu.inflight, s) } }() @@ -537,7 +544,7 @@ func (d *diskHealthCheckingFS) timeFilesystemOp(name string, opType OpType, op f op() // Signal completion by zeroing the start time. - atomic.StoreInt64(&s.startNanos, 0) + s.startNanos.Store(0) } // startTickerLocked starts a new goroutine with a ticker to monitor disk @@ -548,7 +555,12 @@ func (d *diskHealthCheckingFS) startTickerLocked() { go func() { ticker := time.NewTicker(d.tickInterval) defer ticker.Stop() - var exceededSlots []slot + type exceededSlot struct { + name string + opType OpType + startNanos int64 + } + var exceededSlots []exceededSlot for { select { @@ -559,14 +571,15 @@ func (d *diskHealthCheckingFS) startTickerLocked() { d.mu.Lock() now := time.Now() for i := range d.mu.inflight { - nanos := atomic.LoadInt64(&d.mu.inflight[i].startNanos) + nanos := d.mu.inflight[i].startNanos.Load() if nanos != 0 && time.Unix(0, nanos).Add(d.diskSlowThreshold).Before(now) { // diskSlowThreshold was exceeded. Copy this inflightOp into // exceededSlots and call d.onSlowDisk after dropping the mutex. - var inflightOp slot - inflightOp.name = d.mu.inflight[i].name - inflightOp.opType = d.mu.inflight[i].opType - inflightOp.startNanos = nanos + inflightOp := exceededSlot{ + name: d.mu.inflight[i].name, + opType: d.mu.inflight[i].opType, + startNanos: nanos, + } exceededSlots = append(exceededSlots, inflightOp) } } @@ -680,6 +693,11 @@ func (d *diskHealthCheckingFS) Open(name string, opts ...OpenOption) (File, erro return d.fs.Open(name, opts...) } +// OpenReadWrite implements the FS interface. +func (d *diskHealthCheckingFS) OpenReadWrite(name string, opts ...OpenOption) (File, error) { + return d.fs.OpenReadWrite(name, opts...) +} + // OpenDir implements the FS interface. func (d *diskHealthCheckingFS) OpenDir(name string) (File, error) { f, err := d.fs.OpenDir(name) diff --git a/vendor/github.com/cockroachdb/pebble/vfs/disk_usage_openbsd.go b/vendor/github.com/cockroachdb/pebble/vfs/disk_usage_openbsd.go new file mode 100644 index 0000000..19e9c5c --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/vfs/disk_usage_openbsd.go @@ -0,0 +1,26 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +//go:build openbsd +// +build openbsd + +package vfs + +import "golang.org/x/sys/unix" + +func (defaultFS) GetDiskUsage(path string) (DiskUsage, error) { + stat := unix.Statfs_t{} + if err := unix.Statfs(path, &stat); err != nil { + return DiskUsage{}, err + } + + freeBytes := uint64(stat.F_bsize) * uint64(stat.F_bfree) + availBytes := uint64(stat.F_bsize) * uint64(stat.F_bavail) + totalBytes := uint64(stat.F_bsize) * uint64(stat.F_blocks) + return DiskUsage{ + AvailBytes: availBytes, + TotalBytes: totalBytes, + UsedBytes: totalBytes - freeBytes, + }, nil +} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/disk_usage_unix.go b/vendor/github.com/cockroachdb/pebble/vfs/disk_usage_unix.go index 9990241..95d9a7c 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/disk_usage_unix.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/disk_usage_unix.go @@ -2,8 +2,8 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -//go:build darwin || openbsd || dragonfly || freebsd -// +build darwin openbsd dragonfly freebsd +//go:build darwin || dragonfly || freebsd +// +build darwin dragonfly freebsd package vfs diff --git a/vendor/github.com/cockroachdb/pebble/vfs/logging_fs.go b/vendor/github.com/cockroachdb/pebble/vfs/logging_fs.go index ee0c3e2..820c081 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/logging_fs.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/logging_fs.go @@ -37,6 +37,24 @@ func (fs *loggingFS) Create(name string) (File, error) { return newLoggingFile(f, name, fs.logFn), nil } +func (fs *loggingFS) Open(name string, opts ...OpenOption) (File, error) { + fs.logFn("open: %s", name) + f, err := fs.FS.Open(name, opts...) + if err != nil { + return nil, err + } + return newLoggingFile(f, name, fs.logFn), nil +} + +func (fs *loggingFS) OpenReadWrite(name string, opts ...OpenOption) (File, error) { + fs.logFn("open-read-write: %s", name) + f, err := fs.FS.OpenReadWrite(name, opts...) + if err != nil { + return nil, err + } + return newLoggingFile(f, name, fs.logFn), nil +} + func (fs *loggingFS) Link(oldname, newname string) error { fs.logFn("link: %s -> %s", oldname, newname) return fs.FS.Link(oldname, newname) @@ -122,3 +140,18 @@ func (f *loggingFile) SyncTo(length int64) (fullSync bool, err error) { f.logFn("sync-to(%d): %s", length, f.name) return f.File.SyncTo(length) } + +func (f *loggingFile) ReadAt(p []byte, offset int64) (int, error) { + f.logFn("read-at(%d, %d): %s", offset, len(p), f.name) + return f.File.ReadAt(p, offset) +} + +func (f *loggingFile) WriteAt(p []byte, offset int64) (int, error) { + f.logFn("write-at(%d, %d): %s", offset, len(p), f.name) + return f.File.WriteAt(p, offset) +} + +func (f *loggingFile) Prefetch(offset int64, length int64) error { + f.logFn("prefetch(%d, %d): %s", offset, length, f.name) + return f.File.Prefetch(offset, length) +} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/mem_fs.go b/vendor/github.com/cockroachdb/pebble/vfs/mem_fs.go index 5b072da..ef6c87d 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/mem_fs.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/mem_fs.go @@ -14,6 +14,7 @@ import ( "strings" "sync" "sync/atomic" + "syscall" "time" "github.com/cockroachdb/errors" @@ -63,7 +64,8 @@ func NewStrictMem() *MemFS { // NewMemFile returns a memory-backed File implementation. The memory-backed // file takes ownership of data. func NewMemFile(data []byte) File { - n := &memNode{refs: 1} + n := &memNode{} + n.refs.Store(1) n.mu.data = data n.mu.modTime = time.Now() return &memFile{ @@ -77,6 +79,11 @@ type MemFS struct { mu sync.Mutex root *memNode + // lockFiles holds a map of open file locks. Presence in this map indicates + // a file lock is currently held. Keys are strings holding the path of the + // locked file. The stored value is untyped and unused; only presence of + // the key within the map is significant. + lockedFiles sync.Map strict bool ignoreSyncs bool // Windows has peculiar semantics with respect to hard links and deleting @@ -208,6 +215,7 @@ func (y *MemFS) Create(fullname string) (File, error) { ret = &memFile{ n: n, fs: y, + read: true, write: true, } } @@ -216,7 +224,7 @@ func (y *MemFS) Create(fullname string) (File, error) { if err != nil { return nil, err } - atomic.AddInt32(&ret.n.refs, 1) + ret.n.refs.Add(1) return ret, nil } @@ -262,7 +270,7 @@ func (y *MemFS) Link(oldname, newname string) error { }) } -func (y *MemFS) open(fullname string) (File, error) { +func (y *MemFS) open(fullname string, openForWrite bool) (File, error) { var ret *memFile err := y.walk(fullname, func(dir *memNode, frag string, final bool) error { if final { @@ -275,9 +283,10 @@ func (y *MemFS) open(fullname string) (File, error) { } if n := dir.children[frag]; n != nil { ret = &memFile{ - n: n, - fs: y, - read: true, + n: n, + fs: y, + read: true, + write: openForWrite, } } } @@ -293,18 +302,28 @@ func (y *MemFS) open(fullname string) (File, error) { Err: oserror.ErrNotExist, } } - atomic.AddInt32(&ret.n.refs, 1) + ret.n.refs.Add(1) return ret, nil } // Open implements FS.Open. func (y *MemFS) Open(fullname string, opts ...OpenOption) (File, error) { - return y.open(fullname) + return y.open(fullname, false /* openForWrite */) +} + +// OpenReadWrite implements FS.OpenReadWrite. +func (y *MemFS) OpenReadWrite(fullname string, opts ...OpenOption) (File, error) { + f, err := y.open(fullname, true /* openForWrite */) + pathErr, ok := err.(*os.PathError) + if ok && pathErr.Err == oserror.ErrNotExist { + return y.Create(fullname) + } + return f, err } // OpenDir implements FS.OpenDir. func (y *MemFS) OpenDir(fullname string) (File, error) { - return y.open(fullname) + return y.open(fullname, false /* openForWrite */) } // Remove implements FS.Remove. @@ -323,7 +342,7 @@ func (y *MemFS) Remove(fullname string) error { // Windows semantics. This ensures that we don't regress in the // ordering of operations and try to remove a file while it is // still open. - if n := atomic.LoadInt32(&child.refs); n > 0 { + if n := child.refs.Load(); n > 0 { return oserror.ErrInvalid } } @@ -444,9 +463,31 @@ func (y *MemFS) MkdirAll(dirname string, perm os.FileMode) error { // Lock implements FS.Lock. func (y *MemFS) Lock(fullname string) (io.Closer, error) { // FS.Lock excludes other processes, but other processes cannot see this - // process' memory. We translate Lock into Create so that have the normal - // detection of non-existent directory paths. - return y.Create(fullname) + // process' memory. However some uses (eg, Cockroach tests) may open and + // close the same MemFS-backed database multiple times. We want mutual + // exclusion in this case too. See cockroachdb/cockroach#110645. + _, loaded := y.lockedFiles.Swap(fullname, nil /* the value itself is insignificant */) + if loaded { + // This file lock has already been acquired. On unix, this results in + // either EACCES or EAGAIN so we mimic. + return nil, syscall.EAGAIN + } + // Otherwise, we successfully acquired the lock. Locks are visible in the + // parent directory listing, and they also must be created under an existent + // directory. Create the path so that we have the normal detection of + // non-existent directory paths, and make the lock visible when listing + // directory entries. + f, err := y.Create(fullname) + if err != nil { + // "Release" the lock since we failed. + y.lockedFiles.Delete(fullname) + return nil, err + } + return &memFileLock{ + y: y, + f: f, + fullname: fullname, + }, nil } // List implements FS.List. @@ -513,7 +554,7 @@ func (*MemFS) GetDiskUsage(string) (DiskUsage, error) { type memNode struct { name string isDir bool - refs int32 + refs atomic.Int32 // Mutable state. // - For a file: data, syncedDate, modTime: A file is only being mutated by a single goroutine, @@ -629,7 +670,7 @@ type memFile struct { var _ File = (*memFile)(nil) func (f *memFile) Close() error { - if n := atomic.AddInt32(&f.n.refs, -1); n < 0 { + if n := f.n.refs.Add(-1); n < 0 { panic(fmt.Sprintf("pebble: close of unopened file: %d", n)) } f.n = nil @@ -665,7 +706,11 @@ func (f *memFile) ReadAt(p []byte, off int64) (int, error) { if off >= int64(len(f.n.mu.data)) { return 0, io.EOF } - return copy(p, f.n.mu.data[off:]), nil + n := copy(p, f.n.mu.data[off:]) + if n < len(p) { + return n, io.EOF + } + return n, nil } func (f *memFile) Write(p []byte) (int, error) { @@ -698,6 +743,29 @@ func (f *memFile) Write(p []byte) (int, error) { return len(p), nil } +func (f *memFile) WriteAt(p []byte, ofs int64) (int, error) { + if !f.write { + return 0, errors.New("pebble/vfs: file was not created for writing") + } + if f.n.isDir { + return 0, errors.New("pebble/vfs: cannot write a directory") + } + f.n.mu.Lock() + defer f.n.mu.Unlock() + f.n.mu.modTime = time.Now() + + for len(f.n.mu.data) < int(ofs)+len(p) { + f.n.mu.data = append(f.n.mu.data, 0) + } + + n := copy(f.n.mu.data[int(ofs):int(ofs)+len(p)], p) + if n != len(p) { + panic("stuff") + } + + return len(p), nil +} + func (f *memFile) Prefetch(offset int64, length int64) error { return nil } func (f *memFile) Preallocate(offset, length int64) error { return nil } @@ -747,3 +815,18 @@ func (f *memFile) Fd() uintptr { func (f *memFile) Flush() error { return nil } + +type memFileLock struct { + y *MemFS + f File + fullname string +} + +func (l *memFileLock) Close() error { + if l.y == nil { + return nil + } + l.y.lockedFiles.Delete(l.fullname) + l.y = nil + return l.f.Close() +} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/syncing_file.go b/vendor/github.com/cockroachdb/pebble/vfs/syncing_file.go index d1a4ae1..cf59862 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/syncing_file.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/syncing_file.go @@ -26,16 +26,14 @@ type syncingFile struct { noSyncOnClose bool bytesPerSync int64 preallocateSize int64 - atomic struct { - // The offset at which dirty data has been written. - offset int64 - // The offset at which data has been synced. Note that if SyncFileRange is - // being used, the periodic syncing of data during writing will only ever - // sync up to offset-1MB. This is done to avoid rewriting the tail of the - // file multiple times, but has the side effect of ensuring that Close will - // sync the file's metadata. - syncOffset int64 - } + // The offset at which dirty data has been written. + offset atomic.Int64 + // The offset at which data has been synced. Note that if SyncFileRange is + // being used, the periodic syncing of data during writing will only ever + // sync up to offset-1MB. This is done to avoid rewriting the tail of the + // file multiple times, but has the side effect of ensuring that Close will + // sync the file's metadata. + syncOffset atomic.Int64 preallocatedBlocks int64 } @@ -54,13 +52,13 @@ func NewSyncingFile(f File, opts SyncingFileOptions) File { } // Ensure a file that is opened and then closed will be synced, even if no // data has been written to it. - s.atomic.syncOffset = -1 + s.syncOffset.Store(-1) return s } // NB: syncingFile.Write is unsafe for concurrent use! func (f *syncingFile) Write(p []byte) (n int, err error) { - _ = f.preallocate(atomic.LoadInt64(&f.atomic.offset)) + _ = f.preallocate(f.offset.Load()) n, err = f.File.Write(p) if err != nil { @@ -68,7 +66,7 @@ func (f *syncingFile) Write(p []byte) (n int, err error) { } // The offset is updated atomically so that it can be accessed safely from // Sync. - atomic.AddInt64(&f.atomic.offset, int64(n)) + f.offset.Add(int64(n)) if err := f.maybeSync(); err != nil { return 0, err } @@ -93,11 +91,11 @@ func (f *syncingFile) preallocate(offset int64) error { func (f *syncingFile) ratchetSyncOffset(offset int64) { for { - syncOffset := atomic.LoadInt64(&f.atomic.syncOffset) + syncOffset := f.syncOffset.Load() if syncOffset >= offset { return } - if atomic.CompareAndSwapInt64(&f.atomic.syncOffset, syncOffset, offset) { + if f.syncOffset.CompareAndSwap(syncOffset, offset) { return } } @@ -109,7 +107,7 @@ func (f *syncingFile) Sync() error { // offset, we still need to call the underlying file's sync for persistence // guarantees which are not provided by SyncTo (or by sync_file_range on // Linux). - f.ratchetSyncOffset(atomic.LoadInt64(&f.atomic.offset)) + f.ratchetSyncOffset(f.offset.Load()) return f.SyncData() } @@ -127,7 +125,7 @@ func (f *syncingFile) maybeSync() error { // Xfs does neighbor page flushing outside of the specified ranges. We // need to make sure sync range is far from the write offset. const syncRangeBuffer = 1 << 20 // 1 MB - offset := atomic.LoadInt64(&f.atomic.offset) + offset := f.offset.Load() if offset <= syncRangeBuffer { return nil } @@ -135,7 +133,7 @@ func (f *syncingFile) maybeSync() error { const syncRangeAlignment = 4 << 10 // 4 KB syncToOffset := offset - syncRangeBuffer syncToOffset -= syncToOffset % syncRangeAlignment - syncOffset := atomic.LoadInt64(&f.atomic.syncOffset) + syncOffset := f.syncOffset.Load() if syncToOffset < 0 || (syncToOffset-syncOffset) < f.bytesPerSync { return nil } @@ -167,7 +165,7 @@ func (f *syncingFile) Close() error { // caller has not called Sync since the last write, syncOffset is guaranteed // to be less than atomic.offset. This ensures we fall into the below // conditional and perform a full sync to durably persist the file. - if off := atomic.LoadInt64(&f.atomic.offset); off > atomic.LoadInt64(&f.atomic.syncOffset) { + if off := f.offset.Load(); off > f.syncOffset.Load() { // There's still remaining dirty data. if f.noSyncOnClose { diff --git a/vendor/github.com/cockroachdb/pebble/vfs/vfs.go b/vendor/github.com/cockroachdb/pebble/vfs/vfs.go index 7eafbf6..3cc0b43 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/vfs.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/vfs.go @@ -29,6 +29,8 @@ type File interface { // method *is* allowed to modify the slice passed in, whether temporarily // or permanently. Callers of Write() need to take this into account. io.Writer + // WriteAt() is only supported for files that were opened with FS.OpenReadWrite. + io.WriterAt // Preallocate optionally preallocates storage for `length` at `offset` // within the file. Implementations may choose to do nothing. @@ -99,6 +101,10 @@ type FS interface { // Open opens the named file for reading. openOptions provides Open(name string, opts ...OpenOption) (File, error) + // OpenReadWrite opens the named file for reading and writing. If the file + // does not exist, it is created. + OpenReadWrite(name string, opts ...OpenOption) (File, error) + // OpenDir opens the named directory for syncing. OpenDir(name string) (File, error) @@ -232,6 +238,18 @@ func (defaultFS) Open(name string, opts ...OpenOption) (File, error) { return file, nil } +func (defaultFS) OpenReadWrite(name string, opts ...OpenOption) (File, error) { + osFile, err := os.OpenFile(name, os.O_RDWR|syscall.O_CLOEXEC|os.O_CREATE, 0666) + if err != nil { + return nil, errors.WithStack(err) + } + file := wrapOSFile(osFile) + for _, opt := range opts { + opt.Apply(file) + } + return file, nil +} + func (defaultFS) Remove(name string) error { return errors.WithStack(os.Remove(name)) } diff --git a/vendor/github.com/cockroachdb/tokenbucket/LICENSE b/vendor/github.com/cockroachdb/tokenbucket/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/vendor/github.com/cockroachdb/tokenbucket/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/cockroachdb/tokenbucket/README.md b/vendor/github.com/cockroachdb/tokenbucket/README.md new file mode 100644 index 0000000..edf49ba --- /dev/null +++ b/vendor/github.com/cockroachdb/tokenbucket/README.md @@ -0,0 +1,2 @@ +# tokenbucket +Token bucket implementation in Go diff --git a/vendor/github.com/cockroachdb/tokenbucket/token_bucket.go b/vendor/github.com/cockroachdb/tokenbucket/token_bucket.go new file mode 100644 index 0000000..6a8159f --- /dev/null +++ b/vendor/github.com/cockroachdb/tokenbucket/token_bucket.go @@ -0,0 +1,202 @@ +// Copyright 2023 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package tokenbucket + +import ( + "context" + "time" +) + +// Tokens are abstract units (usually units of work). +type Tokens float64 + +// TokensPerSecond is the rate of token replenishment. +type TokensPerSecond float64 + +// TokenBucket implements the basic accounting for a token bucket. +// +// A token bucket has a rate of replenishment and a burst limit. Tokens are +// replenished over time, up to the burst limit. +// +// The token bucket keeps track of the current amount and updates it as time +// passes. The bucket can go into debt (i.e. negative current amount). +type TokenBucket struct { + rate TokensPerSecond + burst Tokens + nowFn func() time.Time + + current Tokens + lastUpdated time.Time + + exhaustedStart time.Time + exhaustedMicros int64 +} + +// Init the token bucket. +func (tb *TokenBucket) Init(rate TokensPerSecond, burst Tokens) { + tb.InitWithNowFn(rate, burst, func() time.Time { + return time.Now() + }) +} + +// Init the token bucket with a custom "Now" function. +// Note that Wait/WaitCtx cannot be used with a custom time function. +func (tb *TokenBucket) InitWithNowFn(rate TokensPerSecond, burst Tokens, nowFn func() time.Time) { + *tb = TokenBucket{ + rate: rate, + burst: burst, + nowFn: nowFn, + current: burst, + lastUpdated: nowFn(), + } +} + +// Update moves the time forward, accounting for the replenishment since the +// last update. +func (tb *TokenBucket) Update() { + now := tb.nowFn() + if since := now.Sub(tb.lastUpdated); since > 0 { + tb.current += Tokens(float64(tb.rate) * since.Seconds()) + + if tb.current > tb.burst { + tb.current = tb.burst + } + tb.lastUpdated = now + tb.updateExhaustedMicros() + } +} + +// UpdateConfig updates the rate and burst limits. The change in burst will be +// applied to the current token quantity. For example, if the RateLimiter +// currently had 5 available tokens and the burst is updated from 10 to 20, the +// amount will increase to 15. Similarly, if the burst is decreased by 10, the +// current quota will decrease accordingly, potentially putting the limiter into +// debt. +func (tb *TokenBucket) UpdateConfig(rate TokensPerSecond, burst Tokens) { + tb.Update() + + burstDelta := burst - tb.burst + tb.rate = rate + tb.burst = burst + + tb.current += burstDelta + tb.updateExhaustedMicros() +} + +// Reset resets the current tokens to whatever the burst is. +func (tb *TokenBucket) Reset() { + tb.current = tb.burst + tb.updateExhaustedMicros() +} + +// Adjust returns tokens to the bucket (positive delta) or accounts for a debt +// of tokens (negative delta). +func (tb *TokenBucket) Adjust(delta Tokens) { + tb.Update() + tb.current += delta + if tb.current > tb.burst { + tb.current = tb.burst + } + tb.updateExhaustedMicros() +} + +// TryToFulfill either removes the given amount if is available, or returns a +// time after which the request should be retried. +func (tb *TokenBucket) TryToFulfill(amount Tokens) (fulfilled bool, tryAgainAfter time.Duration) { + tb.Update() + + // Deal with the case where the request is larger than the burst size. In + // this case we'll allow the acquisition to complete if and when the current + // value is equal to the burst. If the acquisition succeeds, it will put the + // limiter into debt. + want := amount + if want > tb.burst { + want = tb.burst + } + if delta := want - tb.current; delta > 0 { + // Compute the time it will take to get to the needed capacity. + timeDelta := time.Duration((float64(delta) * float64(time.Second)) / float64(tb.rate)) + if timeDelta < time.Nanosecond { + timeDelta = time.Nanosecond + } + return false, timeDelta + } + + tb.current -= amount + tb.updateExhaustedMicros() + return true, 0 +} + +// Wait removes the given amount, waiting as long as necessary. +func (tb *TokenBucket) Wait(amount Tokens) { + _ = tb.WaitCtx(context.Background(), amount) +} + +// WaitCtx removes the given amount, waiting as long as necessary or until the +// context is canceled. +func (tb *TokenBucket) WaitCtx(ctx context.Context, amount Tokens) error { + // We want to check for context cancelation even if we don't need to wait. + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + for { + fulfilled, tryAgainAfter := tb.TryToFulfill(amount) + if fulfilled { + return nil + } + select { + case <-time.After(tryAgainAfter): + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// Exhausted returns the cumulative duration over which this token bucket was +// exhausted. Exported only for metrics. +func (tb *TokenBucket) Exhausted() time.Duration { + var ongoingExhaustionMicros int64 + if !tb.exhaustedStart.IsZero() { + ongoingExhaustionMicros = tb.nowFn().Sub(tb.exhaustedStart).Microseconds() + } + return time.Duration(tb.exhaustedMicros+ongoingExhaustionMicros) * time.Microsecond +} + +// Available returns the currently available tokens (can be -ve). Exported only +// for metrics. +func (tb *TokenBucket) Available() Tokens { + return tb.current +} + +func (tb *TokenBucket) updateExhaustedMicros() { + now := tb.nowFn() + if tb.current <= 0 && tb.exhaustedStart.IsZero() { + tb.exhaustedStart = now + } + if tb.current > 0 && !tb.exhaustedStart.IsZero() { + tb.exhaustedMicros += now.Sub(tb.exhaustedStart).Microseconds() + tb.exhaustedStart = time.Time{} + } +} + +// TestingInternalParameters returns the refill rate (configured), burst tokens +// (configured), and number of available tokens where available <= burst. It's +// used in tests. +func (tb *TokenBucket) TestingInternalParameters() (rate TokensPerSecond, burst, available Tokens) { + tb.Update() + return tb.rate, tb.burst, tb.current +} diff --git a/vendor/github.com/getsentry/sentry-go/CHANGELOG.md b/vendor/github.com/getsentry/sentry-go/CHANGELOG.md index 303945b..4d26707 100644 --- a/vendor/github.com/getsentry/sentry-go/CHANGELOG.md +++ b/vendor/github.com/getsentry/sentry-go/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.27.0 + +The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.27.0. + +### Breaking Changes + +- `Exception.ThreadId` is now typed as `uint64`. It was wrongly typed as `string` before. ([#770](https://github.com/getsentry/sentry-go/pull/770)) + +### Misc + +- Export `Event.Attachments` ([#771](https://github.com/getsentry/sentry-go/pull/771)) + ## 0.26.0 The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.26.0. diff --git a/vendor/github.com/getsentry/sentry-go/interfaces.go b/vendor/github.com/getsentry/sentry-go/interfaces.go index 13517f6..c769faa 100644 --- a/vendor/github.com/getsentry/sentry-go/interfaces.go +++ b/vendor/github.com/getsentry/sentry-go/interfaces.go @@ -241,7 +241,7 @@ type Exception struct { Type string `json:"type,omitempty"` // used as the main issue title Value string `json:"value,omitempty"` // used as the main issue subtitle Module string `json:"module,omitempty"` - ThreadID string `json:"thread_id,omitempty"` + ThreadID uint64 `json:"thread_id,omitempty"` Stacktrace *Stacktrace `json:"stacktrace,omitempty"` Mechanism *Mechanism `json:"mechanism,omitempty"` } @@ -317,6 +317,7 @@ type Event struct { Request *Request `json:"request,omitempty"` Exception []Exception `json:"exception,omitempty"` DebugMeta *DebugMeta `json:"debug_meta,omitempty"` + Attachments []*Attachment `json:"-"` // The fields below are only relevant for transactions. @@ -333,7 +334,6 @@ type Event struct { // The fields below are not part of the final JSON payload. sdkMetaData SDKMetaData - attachments []*Attachment } // SetException appends the unwrapped errors to the event's exception list. diff --git a/vendor/github.com/getsentry/sentry-go/scope.go b/vendor/github.com/getsentry/sentry-go/scope.go index 2f961ba..374010b 100644 --- a/vendor/github.com/getsentry/sentry-go/scope.go +++ b/vendor/github.com/getsentry/sentry-go/scope.go @@ -344,7 +344,7 @@ func (scope *Scope) ApplyToEvent(event *Event, hint *EventHint) *Event { } if len(scope.attachments) > 0 { - event.attachments = append(event.attachments, scope.attachments...) + event.Attachments = append(event.Attachments, scope.attachments...) } if len(scope.tags) > 0 { diff --git a/vendor/github.com/getsentry/sentry-go/sentry.go b/vendor/github.com/getsentry/sentry-go/sentry.go index c44f906..3df2cdb 100644 --- a/vendor/github.com/getsentry/sentry-go/sentry.go +++ b/vendor/github.com/getsentry/sentry-go/sentry.go @@ -6,7 +6,7 @@ import ( ) // The version of the SDK. -const SDKVersion = "0.26.0" +const SDKVersion = "0.27.0" // apiVersion is the minimum version of the Sentry API compatible with the // sentry-go SDK. diff --git a/vendor/github.com/getsentry/sentry-go/transport.go b/vendor/github.com/getsentry/sentry-go/transport.go index 3eee458..0d03462 100644 --- a/vendor/github.com/getsentry/sentry-go/transport.go +++ b/vendor/github.com/getsentry/sentry-go/transport.go @@ -185,7 +185,7 @@ func envelopeFromBody(event *Event, dsn *Dsn, sentAt time.Time, body json.RawMes } // Attachments - for _, attachment := range event.attachments { + for _, attachment := range event.Attachments { if err := encodeAttachment(enc, &b, attachment); err != nil { return nil, err } diff --git a/vendor/github.com/klauspost/compress/.goreleaser.yml b/vendor/github.com/klauspost/compress/.goreleaser.yml index 4c28dff..a229538 100644 --- a/vendor/github.com/klauspost/compress/.goreleaser.yml +++ b/vendor/github.com/klauspost/compress/.goreleaser.yml @@ -3,7 +3,6 @@ before: hooks: - ./gen.sh - - go install mvdan.cc/garble@v0.10.1 builds: - @@ -32,7 +31,6 @@ builds: - mips64le goarm: - 7 - gobinary: garble - id: "s2d" binary: s2d @@ -59,7 +57,6 @@ builds: - mips64le goarm: - 7 - gobinary: garble - id: "s2sx" binary: s2sx @@ -87,7 +84,6 @@ builds: - mips64le goarm: - 7 - gobinary: garble archives: - diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index 7e83f58..1f72cdd 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -16,6 +16,30 @@ This package provides various compression algorithms. # changelog +* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6) + * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923 + * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925 + +* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5) + * flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912 + * zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908 + * zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913 + * zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910 + * s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917 +https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918 + +* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4) + * huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887 + * huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886 + * gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892 + * gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890 + * gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891 + +* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3) + * fse: Fix max header size https://github.com/klauspost/compress/pull/881 + * zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877 + * gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883 + * Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2) * zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876 @@ -554,7 +578,7 @@ For direct deflate use, NewStatelessWriter and StatelessDeflate are available. S A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer: -``` +```go // replace 'ioutil.Discard' with your output. gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression) if err != nil { diff --git a/vendor/github.com/klauspost/compress/s2sx.mod b/vendor/github.com/klauspost/compress/s2sx.mod index 2263853..5a4412f 100644 --- a/vendor/github.com/klauspost/compress/s2sx.mod +++ b/vendor/github.com/klauspost/compress/s2sx.mod @@ -1,4 +1,4 @@ module github.com/klauspost/compress -go 1.16 +go 1.19 diff --git a/vendor/github.com/klauspost/compress/zstd/decodeheader.go b/vendor/github.com/klauspost/compress/zstd/decodeheader.go index f6a2409..6a5a298 100644 --- a/vendor/github.com/klauspost/compress/zstd/decodeheader.go +++ b/vendor/github.com/klauspost/compress/zstd/decodeheader.go @@ -95,42 +95,54 @@ type Header struct { // If there isn't enough input, io.ErrUnexpectedEOF is returned. // The FirstBlock.OK will indicate if enough information was available to decode the first block header. func (h *Header) Decode(in []byte) error { + _, err := h.DecodeAndStrip(in) + return err +} + +// DecodeAndStrip will decode the header from the beginning of the stream +// and on success return the remaining bytes. +// This will decode the frame header and the first block header if enough bytes are provided. +// It is recommended to provide at least HeaderMaxSize bytes. +// If the frame header cannot be read an error will be returned. +// If there isn't enough input, io.ErrUnexpectedEOF is returned. +// The FirstBlock.OK will indicate if enough information was available to decode the first block header. +func (h *Header) DecodeAndStrip(in []byte) (remain []byte, err error) { *h = Header{} if len(in) < 4 { - return io.ErrUnexpectedEOF + return nil, io.ErrUnexpectedEOF } h.HeaderSize += 4 b, in := in[:4], in[4:] if string(b) != frameMagic { if string(b[1:4]) != skippableFrameMagic || b[0]&0xf0 != 0x50 { - return ErrMagicMismatch + return nil, ErrMagicMismatch } if len(in) < 4 { - return io.ErrUnexpectedEOF + return nil, io.ErrUnexpectedEOF } h.HeaderSize += 4 h.Skippable = true h.SkippableID = int(b[0] & 0xf) h.SkippableSize = binary.LittleEndian.Uint32(in) - return nil + return in[4:], nil } // Read Window_Descriptor // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor if len(in) < 1 { - return io.ErrUnexpectedEOF + return nil, io.ErrUnexpectedEOF } fhd, in := in[0], in[1:] h.HeaderSize++ h.SingleSegment = fhd&(1<<5) != 0 h.HasCheckSum = fhd&(1<<2) != 0 if fhd&(1<<3) != 0 { - return errors.New("reserved bit set on frame header") + return nil, errors.New("reserved bit set on frame header") } if !h.SingleSegment { if len(in) < 1 { - return io.ErrUnexpectedEOF + return nil, io.ErrUnexpectedEOF } var wd byte wd, in = in[0], in[1:] @@ -148,7 +160,7 @@ func (h *Header) Decode(in []byte) error { size = 4 } if len(in) < int(size) { - return io.ErrUnexpectedEOF + return nil, io.ErrUnexpectedEOF } b, in = in[:size], in[size:] h.HeaderSize += int(size) @@ -178,7 +190,7 @@ func (h *Header) Decode(in []byte) error { if fcsSize > 0 { h.HasFCS = true if len(in) < fcsSize { - return io.ErrUnexpectedEOF + return nil, io.ErrUnexpectedEOF } b, in = in[:fcsSize], in[fcsSize:] h.HeaderSize += int(fcsSize) @@ -199,7 +211,7 @@ func (h *Header) Decode(in []byte) error { // Frame Header done, we will not fail from now on. if len(in) < 3 { - return nil + return in, nil } tmp := in[:3] bh := uint32(tmp[0]) | (uint32(tmp[1]) << 8) | (uint32(tmp[2]) << 16) @@ -209,7 +221,7 @@ func (h *Header) Decode(in []byte) error { cSize := int(bh >> 3) switch blockType { case blockTypeReserved: - return nil + return in, nil case blockTypeRLE: h.FirstBlock.Compressed = true h.FirstBlock.DecompressedSize = cSize @@ -225,5 +237,25 @@ func (h *Header) Decode(in []byte) error { } h.FirstBlock.OK = true - return nil + return in, nil +} + +// AppendTo will append the encoded header to the dst slice. +// There is no error checking performed on the header values. +func (h *Header) AppendTo(dst []byte) ([]byte, error) { + if h.Skippable { + magic := [4]byte{0x50, 0x2a, 0x4d, 0x18} + magic[0] |= byte(h.SkippableID & 0xf) + dst = append(dst, magic[:]...) + f := h.SkippableSize + return append(dst, uint8(f), uint8(f>>8), uint8(f>>16), uint8(f>>24)), nil + } + f := frameHeader{ + ContentSize: h.FrameContentSize, + WindowSize: uint32(h.WindowSize), + SingleSegment: h.SingleSegment, + Checksum: h.HasCheckSum, + DictID: h.DictionaryID, + } + return f.appendTo(dst), nil } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go index c81a153..87f4287 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_best.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go @@ -201,14 +201,6 @@ encodeLoop: if delta >= e.maxMatchOff || delta <= 0 || load3232(src, offset) != first { return } - if debugAsserts { - if offset >= s { - panic(fmt.Sprintf("offset: %d - s:%d - rep: %d - cur :%d - max: %d", offset, s, rep, e.cur, e.maxMatchOff)) - } - if !bytes.Equal(src[s:s+4], src[offset:offset+4]) { - panic(fmt.Sprintf("first match mismatch: %v != %v, first: %08x", src[s:s+4], src[offset:offset+4], first)) - } - } // Try to quick reject if we already have a long match. if m.length > 16 { left := len(src) - int(m.s+m.length) @@ -227,8 +219,10 @@ encodeLoop: } } l := 4 + e.matchlen(s+4, offset+4, src) - if true { + if m.rep <= 0 { // Extend candidate match backwards as far as possible. + // Do not extend repeats as we can assume they are optimal + // and offsets change if s == nextEmit. tMin := s - e.maxMatchOff if tMin < 0 { tMin = 0 @@ -239,7 +233,14 @@ encodeLoop: l++ } } - + if debugAsserts { + if offset >= s { + panic(fmt.Sprintf("offset: %d - s:%d - rep: %d - cur :%d - max: %d", offset, s, rep, e.cur, e.maxMatchOff)) + } + if !bytes.Equal(src[s:s+l], src[offset:offset+l]) { + panic(fmt.Sprintf("second match mismatch: %v != %v, first: %08x", src[s:s+4], src[offset:offset+4], first)) + } + } cand := match{offset: offset, s: s, length: l, rep: rep} cand.estBits(bitsPerByte) if m.est >= highScore || cand.est-m.est+(cand.s-m.s)*bitsPerByte>>10 < 0 { @@ -336,24 +337,31 @@ encodeLoop: } if debugAsserts { + if best.offset >= best.s { + panic(fmt.Sprintf("best.offset > s: %d >= %d", best.offset, best.s)) + } + if best.s < nextEmit { + panic(fmt.Sprintf("s %d < nextEmit %d", best.s, nextEmit)) + } + if best.offset < s-e.maxMatchOff { + panic(fmt.Sprintf("best.offset < s-e.maxMatchOff: %d < %d", best.offset, s-e.maxMatchOff)) + } if !bytes.Equal(src[best.s:best.s+best.length], src[best.offset:best.offset+best.length]) { panic(fmt.Sprintf("match mismatch: %v != %v", src[best.s:best.s+best.length], src[best.offset:best.offset+best.length])) } } // We have a match, we can store the forward value + s = best.s if best.rep > 0 { var seq seq seq.matchLen = uint32(best.length - zstdMinMatch) - if debugAsserts && s < nextEmit { - panic("s < nextEmit") - } addLiterals(&seq, best.s) // Repeat. If bit 4 is set, this is a non-lit repeat. seq.offset = uint32(best.rep & 3) if debugSequences { - println("repeat sequence", seq, "next s:", s) + println("repeat sequence", seq, "next s:", best.s, "off:", best.s-best.offset) } blk.sequences = append(blk.sequences, seq) @@ -396,7 +404,6 @@ encodeLoop: // A 4-byte match has been found. Update recent offsets. // We'll later see if more than 4 bytes. - s = best.s t := best.offset offset1, offset2, offset3 = s-t, offset1, offset2 diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index faaf819..20671dc 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -94,7 +94,7 @@ func WithEncoderConcurrency(n int) EOption { // The value must be a power of two between MinWindowSize and MaxWindowSize. // A larger value will enable better compression but allocate more memory and, // for above-default values, take considerably longer. -// The default value is determined by the compression level. +// The default value is determined by the compression level and max 8MB. func WithWindowSize(n int) EOption { return func(o *encoderOptions) error { switch { @@ -232,9 +232,9 @@ func WithEncoderLevel(l EncoderLevel) EOption { case SpeedDefault: o.windowSize = 8 << 20 case SpeedBetterCompression: - o.windowSize = 16 << 20 + o.windowSize = 8 << 20 case SpeedBestCompression: - o.windowSize = 32 << 20 + o.windowSize = 8 << 20 } } if !o.customALEntropy { diff --git a/vendor/github.com/klauspost/compress/zstd/frameenc.go b/vendor/github.com/klauspost/compress/zstd/frameenc.go index 2f5d5ed..667ca06 100644 --- a/vendor/github.com/klauspost/compress/zstd/frameenc.go +++ b/vendor/github.com/klauspost/compress/zstd/frameenc.go @@ -76,7 +76,7 @@ func (f frameHeader) appendTo(dst []byte) []byte { if f.SingleSegment { dst = append(dst, uint8(f.ContentSize)) } - // Unless SingleSegment is set, framessizes < 256 are nto stored. + // Unless SingleSegment is set, framessizes < 256 are not stored. case 1: f.ContentSize -= 256 dst = append(dst, uint8(f.ContentSize), uint8(f.ContentSize>>8)) diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go index 332e51f..8adfebb 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go @@ -20,10 +20,9 @@ func (s *fseDecoder) buildDtable() error { if v == -1 { s.dt[highThreshold].setAddBits(uint8(i)) highThreshold-- - symbolNext[i] = 1 - } else { - symbolNext[i] = uint16(v) + v = 1 } + symbolNext[i] = uint16(v) } } @@ -35,10 +34,12 @@ func (s *fseDecoder) buildDtable() error { for ss, v := range s.norm[:s.symbolLen] { for i := 0; i < int(v); i++ { s.dt[position].setAddBits(uint8(ss)) - position = (position + step) & tableMask - for position > highThreshold { + for { // lowprob area position = (position + step) & tableMask + if position <= highThreshold { + break + } } } } diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 974b997..5b06174 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -157,8 +157,7 @@ sequenceDecs_decode_amd64_ll_update_zero: // Update Literal Length State MOVBQZX DI, R14 - SHRQ $0x10, DI - MOVWQZX DI, DI + SHRL $0x10, DI LEAQ (BX)(R14*1), CX MOVQ DX, R15 MOVQ CX, BX @@ -177,8 +176,7 @@ sequenceDecs_decode_amd64_ll_update_zero: // Update Match Length State MOVBQZX R8, R14 - SHRQ $0x10, R8 - MOVWQZX R8, R8 + SHRL $0x10, R8 LEAQ (BX)(R14*1), CX MOVQ DX, R15 MOVQ CX, BX @@ -197,8 +195,7 @@ sequenceDecs_decode_amd64_ll_update_zero: // Update Offset State MOVBQZX R9, R14 - SHRQ $0x10, R9 - MOVWQZX R9, R9 + SHRL $0x10, R9 LEAQ (BX)(R14*1), CX MOVQ DX, R15 MOVQ CX, BX @@ -459,8 +456,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: // Update Literal Length State MOVBQZX DI, R14 - SHRQ $0x10, DI - MOVWQZX DI, DI + SHRL $0x10, DI LEAQ (BX)(R14*1), CX MOVQ DX, R15 MOVQ CX, BX @@ -479,8 +475,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: // Update Match Length State MOVBQZX R8, R14 - SHRQ $0x10, R8 - MOVWQZX R8, R8 + SHRL $0x10, R8 LEAQ (BX)(R14*1), CX MOVQ DX, R15 MOVQ CX, BX @@ -499,8 +494,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: // Update Offset State MOVBQZX R9, R14 - SHRQ $0x10, R9 - MOVWQZX R9, R9 + SHRL $0x10, R9 LEAQ (BX)(R14*1), CX MOVQ DX, R15 MOVQ CX, BX @@ -772,11 +766,10 @@ sequenceDecs_decode_bmi2_fill_2_end: BZHIQ R14, R15, R15 // Update Offset State - BZHIQ R8, R15, CX - SHRXQ R8, R15, R15 - MOVQ $0x00001010, R14 - BEXTRQ R14, R8, R8 - ADDQ CX, R8 + BZHIQ R8, R15, CX + SHRXQ R8, R15, R15 + SHRL $0x10, R8 + ADDQ CX, R8 // Load ctx.ofTable MOVQ ctx+16(FP), CX @@ -784,11 +777,10 @@ sequenceDecs_decode_bmi2_fill_2_end: MOVQ (CX)(R8*8), R8 // Update Match Length State - BZHIQ DI, R15, CX - SHRXQ DI, R15, R15 - MOVQ $0x00001010, R14 - BEXTRQ R14, DI, DI - ADDQ CX, DI + BZHIQ DI, R15, CX + SHRXQ DI, R15, R15 + SHRL $0x10, DI + ADDQ CX, DI // Load ctx.mlTable MOVQ ctx+16(FP), CX @@ -796,10 +788,9 @@ sequenceDecs_decode_bmi2_fill_2_end: MOVQ (CX)(DI*8), DI // Update Literal Length State - BZHIQ SI, R15, CX - MOVQ $0x00001010, R14 - BEXTRQ R14, SI, SI - ADDQ CX, SI + BZHIQ SI, R15, CX + SHRL $0x10, SI + ADDQ CX, SI // Load ctx.llTable MOVQ ctx+16(FP), CX @@ -1032,11 +1023,10 @@ sequenceDecs_decode_56_bmi2_fill_end: BZHIQ R14, R15, R15 // Update Offset State - BZHIQ R8, R15, CX - SHRXQ R8, R15, R15 - MOVQ $0x00001010, R14 - BEXTRQ R14, R8, R8 - ADDQ CX, R8 + BZHIQ R8, R15, CX + SHRXQ R8, R15, R15 + SHRL $0x10, R8 + ADDQ CX, R8 // Load ctx.ofTable MOVQ ctx+16(FP), CX @@ -1044,11 +1034,10 @@ sequenceDecs_decode_56_bmi2_fill_end: MOVQ (CX)(R8*8), R8 // Update Match Length State - BZHIQ DI, R15, CX - SHRXQ DI, R15, R15 - MOVQ $0x00001010, R14 - BEXTRQ R14, DI, DI - ADDQ CX, DI + BZHIQ DI, R15, CX + SHRXQ DI, R15, R15 + SHRL $0x10, DI + ADDQ CX, DI // Load ctx.mlTable MOVQ ctx+16(FP), CX @@ -1056,10 +1045,9 @@ sequenceDecs_decode_56_bmi2_fill_end: MOVQ (CX)(DI*8), DI // Update Literal Length State - BZHIQ SI, R15, CX - MOVQ $0x00001010, R14 - BEXTRQ R14, SI, SI - ADDQ CX, SI + BZHIQ SI, R15, CX + SHRL $0x10, SI + ADDQ CX, SI // Load ctx.llTable MOVQ ctx+16(FP), CX @@ -1967,8 +1955,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: // Update Literal Length State MOVBQZX DI, R13 - SHRQ $0x10, DI - MOVWQZX DI, DI + SHRL $0x10, DI LEAQ (BX)(R13*1), CX MOVQ DX, R14 MOVQ CX, BX @@ -1987,8 +1974,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: // Update Match Length State MOVBQZX R8, R13 - SHRQ $0x10, R8 - MOVWQZX R8, R8 + SHRL $0x10, R8 LEAQ (BX)(R13*1), CX MOVQ DX, R14 MOVQ CX, BX @@ -2007,8 +1993,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: // Update Offset State MOVBQZX R9, R13 - SHRQ $0x10, R9 - MOVWQZX R9, R9 + SHRL $0x10, R9 LEAQ (BX)(R13*1), CX MOVQ DX, R14 MOVQ CX, BX @@ -2514,11 +2499,10 @@ sequenceDecs_decodeSync_bmi2_fill_2_end: BZHIQ R13, R14, R14 // Update Offset State - BZHIQ R8, R14, CX - SHRXQ R8, R14, R14 - MOVQ $0x00001010, R13 - BEXTRQ R13, R8, R8 - ADDQ CX, R8 + BZHIQ R8, R14, CX + SHRXQ R8, R14, R14 + SHRL $0x10, R8 + ADDQ CX, R8 // Load ctx.ofTable MOVQ ctx+16(FP), CX @@ -2526,11 +2510,10 @@ sequenceDecs_decodeSync_bmi2_fill_2_end: MOVQ (CX)(R8*8), R8 // Update Match Length State - BZHIQ DI, R14, CX - SHRXQ DI, R14, R14 - MOVQ $0x00001010, R13 - BEXTRQ R13, DI, DI - ADDQ CX, DI + BZHIQ DI, R14, CX + SHRXQ DI, R14, R14 + SHRL $0x10, DI + ADDQ CX, DI // Load ctx.mlTable MOVQ ctx+16(FP), CX @@ -2538,10 +2521,9 @@ sequenceDecs_decodeSync_bmi2_fill_2_end: MOVQ (CX)(DI*8), DI // Update Literal Length State - BZHIQ SI, R14, CX - MOVQ $0x00001010, R13 - BEXTRQ R13, SI, SI - ADDQ CX, SI + BZHIQ SI, R14, CX + SHRL $0x10, SI + ADDQ CX, SI // Load ctx.llTable MOVQ ctx+16(FP), CX @@ -3055,8 +3037,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: // Update Literal Length State MOVBQZX DI, R13 - SHRQ $0x10, DI - MOVWQZX DI, DI + SHRL $0x10, DI LEAQ (BX)(R13*1), CX MOVQ DX, R14 MOVQ CX, BX @@ -3075,8 +3056,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: // Update Match Length State MOVBQZX R8, R13 - SHRQ $0x10, R8 - MOVWQZX R8, R8 + SHRL $0x10, R8 LEAQ (BX)(R13*1), CX MOVQ DX, R14 MOVQ CX, BX @@ -3095,8 +3075,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: // Update Offset State MOVBQZX R9, R13 - SHRQ $0x10, R9 - MOVWQZX R9, R9 + SHRL $0x10, R9 LEAQ (BX)(R13*1), CX MOVQ DX, R14 MOVQ CX, BX @@ -3704,11 +3683,10 @@ sequenceDecs_decodeSync_safe_bmi2_fill_2_end: BZHIQ R13, R14, R14 // Update Offset State - BZHIQ R8, R14, CX - SHRXQ R8, R14, R14 - MOVQ $0x00001010, R13 - BEXTRQ R13, R8, R8 - ADDQ CX, R8 + BZHIQ R8, R14, CX + SHRXQ R8, R14, R14 + SHRL $0x10, R8 + ADDQ CX, R8 // Load ctx.ofTable MOVQ ctx+16(FP), CX @@ -3716,11 +3694,10 @@ sequenceDecs_decodeSync_safe_bmi2_fill_2_end: MOVQ (CX)(R8*8), R8 // Update Match Length State - BZHIQ DI, R14, CX - SHRXQ DI, R14, R14 - MOVQ $0x00001010, R13 - BEXTRQ R13, DI, DI - ADDQ CX, DI + BZHIQ DI, R14, CX + SHRXQ DI, R14, R14 + SHRL $0x10, DI + ADDQ CX, DI // Load ctx.mlTable MOVQ ctx+16(FP), CX @@ -3728,10 +3705,9 @@ sequenceDecs_decodeSync_safe_bmi2_fill_2_end: MOVQ (CX)(DI*8), DI // Update Literal Length State - BZHIQ SI, R14, CX - MOVQ $0x00001010, R13 - BEXTRQ R13, SI, SI - ADDQ CX, SI + BZHIQ SI, R14, CX + SHRL $0x10, SI + ADDQ CX, SI // Load ctx.llTable MOVQ ctx+16(FP), CX diff --git a/vendor/github.com/prometheus/client_model/go/metrics.pb.go b/vendor/github.com/prometheus/client_model/go/metrics.pb.go index cee360d..2f15490 100644 --- a/vendor/github.com/prometheus/client_model/go/metrics.pb.go +++ b/vendor/github.com/prometheus/client_model/go/metrics.pb.go @@ -483,6 +483,8 @@ type Histogram struct { // histograms. PositiveDelta []int64 `protobuf:"zigzag64,13,rep,name=positive_delta,json=positiveDelta" json:"positive_delta,omitempty"` // Count delta of each bucket compared to previous one (or to zero for 1st bucket). PositiveCount []float64 `protobuf:"fixed64,14,rep,name=positive_count,json=positiveCount" json:"positive_count,omitempty"` // Absolute count of each bucket. + // Only used for native histograms. These exemplars MUST have a timestamp. + Exemplars []*Exemplar `protobuf:"bytes,16,rep,name=exemplars" json:"exemplars,omitempty"` } func (x *Histogram) Reset() { @@ -622,6 +624,13 @@ func (x *Histogram) GetPositiveCount() []float64 { return nil } +func (x *Histogram) GetExemplars() []*Exemplar { + if x != nil { + return x.Exemplars + } + return nil +} + // A Bucket of a conventional histogram, each of which is treated as // an individual counter-like time series by Prometheus. type Bucket struct { @@ -923,6 +932,7 @@ type MetricFamily struct { Help *string `protobuf:"bytes,2,opt,name=help" json:"help,omitempty"` Type *MetricType `protobuf:"varint,3,opt,name=type,enum=io.prometheus.client.MetricType" json:"type,omitempty"` Metric []*Metric `protobuf:"bytes,4,rep,name=metric" json:"metric,omitempty"` + Unit *string `protobuf:"bytes,5,opt,name=unit" json:"unit,omitempty"` } func (x *MetricFamily) Reset() { @@ -985,6 +995,13 @@ func (x *MetricFamily) GetMetric() []*Metric { return nil } +func (x *MetricFamily) GetUnit() string { + if x != nil && x.Unit != nil { + return *x.Unit + } + return "" +} + var File_io_prometheus_client_metrics_proto protoreflect.FileDescriptor var file_io_prometheus_client_metrics_proto_rawDesc = []byte{ @@ -1028,7 +1045,7 @@ var file_io_prometheus_client_metrics_proto_rawDesc = []byte{ 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x1f, 0x0a, 0x07, 0x55, 0x6e, 0x74, 0x79, 0x70, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xac, 0x05, 0x0a, 0x09, 0x48, + 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xea, 0x05, 0x0a, 0x09, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x73, @@ -1071,79 +1088,84 @@ var file_io_prometheus_client_metrics_proto_rawDesc = []byte{ 0x03, 0x28, 0x12, 0x52, 0x0d, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x01, 0x52, 0x0d, 0x70, 0x6f, 0x73, 0x69, - 0x74, 0x69, 0x76, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xc6, 0x01, 0x0a, 0x06, 0x42, 0x75, - 0x63, 0x6b, 0x65, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, - 0x76, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, - 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, - 0x34, 0x0a, 0x16, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x6f, - 0x75, 0x6e, 0x74, 0x5f, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x52, - 0x14, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, - 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x75, 0x70, 0x70, 0x65, 0x72, 0x5f, 0x62, - 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x75, 0x70, 0x70, 0x65, - 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x3a, 0x0a, 0x08, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, - 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, - 0x45, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x52, 0x08, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x72, 0x22, 0x3c, 0x0a, 0x0a, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x70, 0x61, 0x6e, - 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x11, - 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, - 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, - 0x22, 0x91, 0x01, 0x0a, 0x08, 0x45, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x12, 0x35, 0x0a, + 0x74, 0x69, 0x76, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x3c, 0x0a, 0x09, 0x65, 0x78, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x73, 0x18, 0x10, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x69, + 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x52, 0x09, 0x65, 0x78, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x73, 0x22, 0xc6, 0x01, 0x0a, 0x06, 0x42, 0x75, 0x63, 0x6b, + 0x65, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, + 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x63, 0x75, + 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x34, 0x0a, + 0x16, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x5f, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x52, 0x14, 0x63, + 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x46, 0x6c, + 0x6f, 0x61, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x75, 0x70, 0x70, 0x65, 0x72, 0x5f, 0x62, 0x6f, 0x75, + 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x75, 0x70, 0x70, 0x65, 0x72, 0x42, + 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x3a, 0x0a, 0x08, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, + 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x78, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x52, 0x08, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, + 0x22, 0x3c, 0x0a, 0x0a, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x70, 0x61, 0x6e, 0x12, 0x16, + 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x11, 0x52, 0x06, + 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0x91, + 0x01, 0x0a, 0x08, 0x45, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x12, 0x35, 0x0a, 0x05, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x69, 0x6f, 0x2e, + 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x50, 0x61, 0x69, 0x72, 0x52, 0x05, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x22, 0xff, 0x02, 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x35, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x50, 0x61, 0x69, 0x72, 0x52, 0x05, 0x6c, - 0x61, 0x62, 0x65, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x22, 0xff, 0x02, 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, - 0x35, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, - 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, - 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x50, 0x61, 0x69, 0x72, 0x52, - 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x31, 0x0a, 0x05, 0x67, 0x61, 0x75, 0x67, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, - 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x61, 0x75, - 0x67, 0x65, 0x52, 0x05, 0x67, 0x61, 0x75, 0x67, 0x65, 0x12, 0x37, 0x0a, 0x07, 0x63, 0x6f, 0x75, - 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x69, 0x6f, 0x2e, + 0x61, 0x62, 0x65, 0x6c, 0x12, 0x31, 0x0a, 0x05, 0x67, 0x61, 0x75, 0x67, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, + 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x47, 0x61, 0x75, 0x67, 0x65, + 0x52, 0x05, 0x67, 0x61, 0x75, 0x67, 0x65, 0x12, 0x37, 0x0a, 0x07, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, + 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, + 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x52, 0x07, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, + 0x12, 0x37, 0x0a, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1d, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, + 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, + 0x52, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x37, 0x0a, 0x07, 0x75, 0x6e, 0x74, + 0x79, 0x70, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x52, 0x07, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x65, 0x72, 0x12, 0x37, 0x0a, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, - 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x75, 0x6d, 0x6d, 0x61, - 0x72, 0x79, 0x52, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x37, 0x0a, 0x07, 0x75, - 0x6e, 0x74, 0x79, 0x70, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x69, - 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, - 0x65, 0x6e, 0x74, 0x2e, 0x55, 0x6e, 0x74, 0x79, 0x70, 0x65, 0x64, 0x52, 0x07, 0x75, 0x6e, 0x74, - 0x79, 0x70, 0x65, 0x64, 0x12, 0x3d, 0x0a, 0x09, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, - 0x6d, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, - 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x48, - 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x52, 0x09, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, - 0x72, 0x61, 0x6d, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x5f, 0x6d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x4d, 0x73, 0x22, 0xa2, 0x01, 0x0a, 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, - 0x65, 0x6c, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x65, 0x6c, 0x70, 0x12, - 0x34, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, - 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x34, 0x0a, 0x06, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, - 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x2a, 0x62, 0x0a, 0x0a, 0x4d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x55, - 0x4e, 0x54, 0x45, 0x52, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x47, 0x41, 0x55, 0x47, 0x45, 0x10, - 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x4d, 0x4d, 0x41, 0x52, 0x59, 0x10, 0x02, 0x12, 0x0b, - 0x0a, 0x07, 0x55, 0x4e, 0x54, 0x59, 0x50, 0x45, 0x44, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x48, - 0x49, 0x53, 0x54, 0x4f, 0x47, 0x52, 0x41, 0x4d, 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x47, 0x41, - 0x55, 0x47, 0x45, 0x5f, 0x48, 0x49, 0x53, 0x54, 0x4f, 0x47, 0x52, 0x41, 0x4d, 0x10, 0x05, 0x42, - 0x52, 0x0a, 0x14, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, - 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2f, 0x63, - 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2f, 0x67, 0x6f, 0x3b, 0x69, - 0x6f, 0x5f, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x5f, 0x63, 0x6c, 0x69, - 0x65, 0x6e, 0x74, + 0x74, 0x2e, 0x55, 0x6e, 0x74, 0x79, 0x70, 0x65, 0x64, 0x52, 0x07, 0x75, 0x6e, 0x74, 0x79, 0x70, + 0x65, 0x64, 0x12, 0x3d, 0x0a, 0x09, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, + 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x69, 0x73, + 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x52, 0x09, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, + 0x6d, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x6d, + 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x4d, 0x73, 0x22, 0xb6, 0x01, 0x0a, 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x46, + 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x6c, + 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x65, 0x6c, 0x70, 0x12, 0x34, 0x0a, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x69, 0x6f, + 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, + 0x79, 0x70, 0x65, 0x12, 0x34, 0x0a, 0x06, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, + 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x6e, 0x69, + 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x6e, 0x69, 0x74, 0x2a, 0x62, 0x0a, + 0x0a, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x43, + 0x4f, 0x55, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x47, 0x41, 0x55, 0x47, + 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x4d, 0x4d, 0x41, 0x52, 0x59, 0x10, 0x02, + 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x54, 0x59, 0x50, 0x45, 0x44, 0x10, 0x03, 0x12, 0x0d, 0x0a, + 0x09, 0x48, 0x49, 0x53, 0x54, 0x4f, 0x47, 0x52, 0x41, 0x4d, 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, + 0x47, 0x41, 0x55, 0x47, 0x45, 0x5f, 0x48, 0x49, 0x53, 0x54, 0x4f, 0x47, 0x52, 0x41, 0x4d, 0x10, + 0x05, 0x42, 0x52, 0x0a, 0x14, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, + 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, + 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2f, 0x67, 0x6f, + 0x3b, 0x69, 0x6f, 0x5f, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x5f, 0x63, + 0x6c, 0x69, 0x65, 0x6e, 0x74, } var ( @@ -1185,22 +1207,23 @@ var file_io_prometheus_client_metrics_proto_depIdxs = []int32{ 13, // 5: io.prometheus.client.Histogram.created_timestamp:type_name -> google.protobuf.Timestamp 9, // 6: io.prometheus.client.Histogram.negative_span:type_name -> io.prometheus.client.BucketSpan 9, // 7: io.prometheus.client.Histogram.positive_span:type_name -> io.prometheus.client.BucketSpan - 10, // 8: io.prometheus.client.Bucket.exemplar:type_name -> io.prometheus.client.Exemplar - 1, // 9: io.prometheus.client.Exemplar.label:type_name -> io.prometheus.client.LabelPair - 13, // 10: io.prometheus.client.Exemplar.timestamp:type_name -> google.protobuf.Timestamp - 1, // 11: io.prometheus.client.Metric.label:type_name -> io.prometheus.client.LabelPair - 2, // 12: io.prometheus.client.Metric.gauge:type_name -> io.prometheus.client.Gauge - 3, // 13: io.prometheus.client.Metric.counter:type_name -> io.prometheus.client.Counter - 5, // 14: io.prometheus.client.Metric.summary:type_name -> io.prometheus.client.Summary - 6, // 15: io.prometheus.client.Metric.untyped:type_name -> io.prometheus.client.Untyped - 7, // 16: io.prometheus.client.Metric.histogram:type_name -> io.prometheus.client.Histogram - 0, // 17: io.prometheus.client.MetricFamily.type:type_name -> io.prometheus.client.MetricType - 11, // 18: io.prometheus.client.MetricFamily.metric:type_name -> io.prometheus.client.Metric - 19, // [19:19] is the sub-list for method output_type - 19, // [19:19] is the sub-list for method input_type - 19, // [19:19] is the sub-list for extension type_name - 19, // [19:19] is the sub-list for extension extendee - 0, // [0:19] is the sub-list for field type_name + 10, // 8: io.prometheus.client.Histogram.exemplars:type_name -> io.prometheus.client.Exemplar + 10, // 9: io.prometheus.client.Bucket.exemplar:type_name -> io.prometheus.client.Exemplar + 1, // 10: io.prometheus.client.Exemplar.label:type_name -> io.prometheus.client.LabelPair + 13, // 11: io.prometheus.client.Exemplar.timestamp:type_name -> google.protobuf.Timestamp + 1, // 12: io.prometheus.client.Metric.label:type_name -> io.prometheus.client.LabelPair + 2, // 13: io.prometheus.client.Metric.gauge:type_name -> io.prometheus.client.Gauge + 3, // 14: io.prometheus.client.Metric.counter:type_name -> io.prometheus.client.Counter + 5, // 15: io.prometheus.client.Metric.summary:type_name -> io.prometheus.client.Summary + 6, // 16: io.prometheus.client.Metric.untyped:type_name -> io.prometheus.client.Untyped + 7, // 17: io.prometheus.client.Metric.histogram:type_name -> io.prometheus.client.Histogram + 0, // 18: io.prometheus.client.MetricFamily.type:type_name -> io.prometheus.client.MetricType + 11, // 19: io.prometheus.client.MetricFamily.metric:type_name -> io.prometheus.client.Metric + 20, // [20:20] is the sub-list for method output_type + 20, // [20:20] is the sub-list for method input_type + 20, // [20:20] is the sub-list for extension type_name + 20, // [20:20] is the sub-list for extension extendee + 0, // [0:20] is the sub-list for field type_name } func init() { file_io_prometheus_client_metrics_proto_init() } diff --git a/vendor/github.com/prometheus/common/expfmt/decode.go b/vendor/github.com/prometheus/common/expfmt/decode.go index a909b17..b2b89b0 100644 --- a/vendor/github.com/prometheus/common/expfmt/decode.go +++ b/vendor/github.com/prometheus/common/expfmt/decode.go @@ -45,7 +45,7 @@ func ResponseFormat(h http.Header) Format { mediatype, params, err := mime.ParseMediaType(ct) if err != nil { - return FmtUnknown + return fmtUnknown } const textType = "text/plain" @@ -53,28 +53,28 @@ func ResponseFormat(h http.Header) Format { switch mediatype { case ProtoType: if p, ok := params["proto"]; ok && p != ProtoProtocol { - return FmtUnknown + return fmtUnknown } if e, ok := params["encoding"]; ok && e != "delimited" { - return FmtUnknown + return fmtUnknown } - return FmtProtoDelim + return fmtProtoDelim case textType: if v, ok := params["version"]; ok && v != TextVersion { - return FmtUnknown + return fmtUnknown } - return FmtText + return fmtText } - return FmtUnknown + return fmtUnknown } // NewDecoder returns a new decoder based on the given input format. // If the input format does not imply otherwise, a text format decoder is returned. func NewDecoder(r io.Reader, format Format) Decoder { - switch format { - case FmtProtoDelim: + switch format.FormatType() { + case TypeProtoDelim: return &protoDecoder{r: r} } return &textDecoder{r: r} diff --git a/vendor/github.com/prometheus/common/expfmt/encode.go b/vendor/github.com/prometheus/common/expfmt/encode.go index 02b7a5e..8fd8061 100644 --- a/vendor/github.com/prometheus/common/expfmt/encode.go +++ b/vendor/github.com/prometheus/common/expfmt/encode.go @@ -22,6 +22,7 @@ import ( "google.golang.org/protobuf/encoding/prototext" "github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" + "github.com/prometheus/common/model" dto "github.com/prometheus/client_model/go" ) @@ -61,23 +62,32 @@ func (ec encoderCloser) Close() error { // as the support is still experimental. To include the option to negotiate // FmtOpenMetrics, use NegotiateOpenMetrics. func Negotiate(h http.Header) Format { + escapingScheme := Format(fmt.Sprintf("; escaping=%s", Format(model.NameEscapingScheme.String()))) for _, ac := range goautoneg.ParseAccept(h.Get(hdrAccept)) { + if escapeParam := ac.Params[model.EscapingKey]; escapeParam != "" { + switch Format(escapeParam) { + case model.AllowUTF8, model.EscapeUnderscores, model.EscapeDots, model.EscapeValues: + escapingScheme = Format(fmt.Sprintf("; escaping=%s", escapeParam)) + default: + // If the escaping parameter is unknown, ignore it. + } + } ver := ac.Params["version"] if ac.Type+"/"+ac.SubType == ProtoType && ac.Params["proto"] == ProtoProtocol { switch ac.Params["encoding"] { case "delimited": - return FmtProtoDelim + return fmtProtoDelim + escapingScheme case "text": - return FmtProtoText + return fmtProtoText + escapingScheme case "compact-text": - return FmtProtoCompact + return fmtProtoCompact + escapingScheme } } if ac.Type == "text" && ac.SubType == "plain" && (ver == TextVersion || ver == "") { - return FmtText + return fmtText + escapingScheme } } - return FmtText + return fmtText + escapingScheme } // NegotiateIncludingOpenMetrics works like Negotiate but includes @@ -85,29 +95,40 @@ func Negotiate(h http.Header) Format { // temporary and will disappear once FmtOpenMetrics is fully supported and as // such may be negotiated by the normal Negotiate function. func NegotiateIncludingOpenMetrics(h http.Header) Format { + escapingScheme := Format(fmt.Sprintf("; escaping=%s", Format(model.NameEscapingScheme.String()))) for _, ac := range goautoneg.ParseAccept(h.Get(hdrAccept)) { + if escapeParam := ac.Params[model.EscapingKey]; escapeParam != "" { + switch Format(escapeParam) { + case model.AllowUTF8, model.EscapeUnderscores, model.EscapeDots, model.EscapeValues: + escapingScheme = Format(fmt.Sprintf("; escaping=%s", escapeParam)) + default: + // If the escaping parameter is unknown, ignore it. + } + } ver := ac.Params["version"] if ac.Type+"/"+ac.SubType == ProtoType && ac.Params["proto"] == ProtoProtocol { switch ac.Params["encoding"] { case "delimited": - return FmtProtoDelim + return fmtProtoDelim + escapingScheme case "text": - return FmtProtoText + return fmtProtoText + escapingScheme case "compact-text": - return FmtProtoCompact + return fmtProtoCompact + escapingScheme } } if ac.Type == "text" && ac.SubType == "plain" && (ver == TextVersion || ver == "") { - return FmtText + return fmtText + escapingScheme } if ac.Type+"/"+ac.SubType == OpenMetricsType && (ver == OpenMetricsVersion_0_0_1 || ver == OpenMetricsVersion_1_0_0 || ver == "") { - if ver == OpenMetricsVersion_1_0_0 { - return FmtOpenMetrics_1_0_0 + switch ver { + case OpenMetricsVersion_1_0_0: + return fmtOpenMetrics_1_0_0 + escapingScheme + default: + return fmtOpenMetrics_0_0_1 + escapingScheme } - return FmtOpenMetrics_0_0_1 } } - return FmtText + return fmtText + escapingScheme } // NewEncoder returns a new encoder based on content type negotiation. All @@ -116,9 +137,13 @@ func NegotiateIncludingOpenMetrics(h http.Header) Format { // for FmtOpenMetrics, but a future (breaking) release will add the Close method // to the Encoder interface directly. The current version of the Encoder // interface is kept for backwards compatibility. +// In cases where the Format does not allow for UTF-8 names, the global +// NameEscapingScheme will be applied. func NewEncoder(w io.Writer, format Format) Encoder { - switch format { - case FmtProtoDelim: + escapingScheme := format.ToEscapingScheme() + + switch format.FormatType() { + case TypeProtoDelim: return encoderCloser{ encode: func(v *dto.MetricFamily) error { _, err := protodelim.MarshalTo(w, v) @@ -126,34 +151,34 @@ func NewEncoder(w io.Writer, format Format) Encoder { }, close: func() error { return nil }, } - case FmtProtoCompact: + case TypeProtoCompact: return encoderCloser{ encode: func(v *dto.MetricFamily) error { - _, err := fmt.Fprintln(w, v.String()) + _, err := fmt.Fprintln(w, model.EscapeMetricFamily(v, escapingScheme).String()) return err }, close: func() error { return nil }, } - case FmtProtoText: + case TypeProtoText: return encoderCloser{ encode: func(v *dto.MetricFamily) error { - _, err := fmt.Fprintln(w, prototext.Format(v)) + _, err := fmt.Fprintln(w, prototext.Format(model.EscapeMetricFamily(v, escapingScheme))) return err }, close: func() error { return nil }, } - case FmtText: + case TypeTextPlain: return encoderCloser{ encode: func(v *dto.MetricFamily) error { - _, err := MetricFamilyToText(w, v) + _, err := MetricFamilyToText(w, model.EscapeMetricFamily(v, escapingScheme)) return err }, close: func() error { return nil }, } - case FmtOpenMetrics_0_0_1, FmtOpenMetrics_1_0_0: + case TypeOpenMetrics: return encoderCloser{ encode: func(v *dto.MetricFamily) error { - _, err := MetricFamilyToOpenMetrics(w, v) + _, err := MetricFamilyToOpenMetrics(w, model.EscapeMetricFamily(v, escapingScheme)) return err }, close: func() error { diff --git a/vendor/github.com/prometheus/common/expfmt/expfmt.go b/vendor/github.com/prometheus/common/expfmt/expfmt.go index c4cb20f..6fc9555 100644 --- a/vendor/github.com/prometheus/common/expfmt/expfmt.go +++ b/vendor/github.com/prometheus/common/expfmt/expfmt.go @@ -14,30 +14,154 @@ // Package expfmt contains tools for reading and writing Prometheus metrics. package expfmt +import ( + "strings" + + "github.com/prometheus/common/model" +) + // Format specifies the HTTP content type of the different wire protocols. type Format string -// Constants to assemble the Content-Type values for the different wire protocols. +// Constants to assemble the Content-Type values for the different wire +// protocols. The Content-Type strings here are all for the legacy exposition +// formats, where valid characters for metric names and label names are limited. +// Support for arbitrary UTF-8 characters in those names is already partially +// implemented in this module (see model.ValidationScheme), but to actually use +// it on the wire, new content-type strings will have to be agreed upon and +// added here. const ( TextVersion = "0.0.4" ProtoType = `application/vnd.google.protobuf` ProtoProtocol = `io.prometheus.client.MetricFamily` - ProtoFmt = ProtoType + "; proto=" + ProtoProtocol + ";" + protoFmt = ProtoType + "; proto=" + ProtoProtocol + ";" OpenMetricsType = `application/openmetrics-text` OpenMetricsVersion_0_0_1 = "0.0.1" OpenMetricsVersion_1_0_0 = "1.0.0" - // The Content-Type values for the different wire protocols. - FmtUnknown Format = `` - FmtText Format = `text/plain; version=` + TextVersion + `; charset=utf-8` - FmtProtoDelim Format = ProtoFmt + ` encoding=delimited` - FmtProtoText Format = ProtoFmt + ` encoding=text` - FmtProtoCompact Format = ProtoFmt + ` encoding=compact-text` - FmtOpenMetrics_1_0_0 Format = OpenMetricsType + `; version=` + OpenMetricsVersion_1_0_0 + `; charset=utf-8` - FmtOpenMetrics_0_0_1 Format = OpenMetricsType + `; version=` + OpenMetricsVersion_0_0_1 + `; charset=utf-8` + // The Content-Type values for the different wire protocols. Note that these + // values are now unexported. If code was relying on comparisons to these + // constants, instead use FormatType(). + fmtUnknown Format = `` + fmtText Format = `text/plain; version=` + TextVersion + `; charset=utf-8` + fmtProtoDelim Format = protoFmt + ` encoding=delimited` + fmtProtoText Format = protoFmt + ` encoding=text` + fmtProtoCompact Format = protoFmt + ` encoding=compact-text` + fmtOpenMetrics_1_0_0 Format = OpenMetricsType + `; version=` + OpenMetricsVersion_1_0_0 + `; charset=utf-8` + fmtOpenMetrics_0_0_1 Format = OpenMetricsType + `; version=` + OpenMetricsVersion_0_0_1 + `; charset=utf-8` ) const ( hdrContentType = "Content-Type" hdrAccept = "Accept" ) + +// FormatType is a Go enum representing the overall category for the given +// Format. As the number of Format permutations increases, doing basic string +// comparisons are not feasible, so this enum captures the most useful +// high-level attribute of the Format string. +type FormatType int + +const ( + TypeUnknown = iota + TypeProtoCompact + TypeProtoDelim + TypeProtoText + TypeTextPlain + TypeOpenMetrics +) + +// NewFormat generates a new Format from the type provided. Mostly used for +// tests, most Formats should be generated as part of content negotiation in +// encode.go. +func NewFormat(t FormatType) Format { + switch t { + case TypeProtoCompact: + return fmtProtoCompact + case TypeProtoDelim: + return fmtProtoDelim + case TypeProtoText: + return fmtProtoText + case TypeTextPlain: + return fmtText + case TypeOpenMetrics: + return fmtOpenMetrics_1_0_0 + default: + return fmtUnknown + } +} + +// FormatType deduces an overall FormatType for the given format. +func (f Format) FormatType() FormatType { + toks := strings.Split(string(f), ";") + if len(toks) < 2 { + return TypeUnknown + } + + params := make(map[string]string) + for i, t := range toks { + if i == 0 { + continue + } + args := strings.Split(t, "=") + if len(args) != 2 { + continue + } + params[strings.TrimSpace(args[0])] = strings.TrimSpace(args[1]) + } + + switch strings.TrimSpace(toks[0]) { + case ProtoType: + if params["proto"] != ProtoProtocol { + return TypeUnknown + } + switch params["encoding"] { + case "delimited": + return TypeProtoDelim + case "text": + return TypeProtoText + case "compact-text": + return TypeProtoCompact + default: + return TypeUnknown + } + case OpenMetricsType: + if params["charset"] != "utf-8" { + return TypeUnknown + } + return TypeOpenMetrics + case "text/plain": + v, ok := params["version"] + if !ok { + return TypeTextPlain + } + if v == TextVersion { + return TypeTextPlain + } + return TypeUnknown + default: + return TypeUnknown + } +} + +// ToEscapingScheme returns an EscapingScheme depending on the Format. Iff the +// Format contains a escaping=allow-utf-8 term, it will select NoEscaping. If a valid +// "escaping" term exists, that will be used. Otherwise, the global default will +// be returned. +func (format Format) ToEscapingScheme() model.EscapingScheme { + for _, p := range strings.Split(string(format), ";") { + toks := strings.Split(p, "=") + if len(toks) != 2 { + continue + } + key, value := strings.TrimSpace(toks[0]), strings.TrimSpace(toks[1]) + if key == model.EscapingKey { + scheme, err := model.ToEscapingScheme(value) + if err != nil { + return model.NameEscapingScheme + } + return scheme + } + } + return model.NameEscapingScheme +} diff --git a/vendor/github.com/prometheus/common/expfmt/openmetrics_create.go b/vendor/github.com/prometheus/common/expfmt/openmetrics_create.go index 21cdddc..5622578 100644 --- a/vendor/github.com/prometheus/common/expfmt/openmetrics_create.go +++ b/vendor/github.com/prometheus/common/expfmt/openmetrics_create.go @@ -35,6 +35,18 @@ import ( // sanity checks. If the input contains duplicate metrics or invalid metric or // label names, the conversion will result in invalid text format output. // +// If metric names conform to the legacy validation pattern, they will be placed +// outside the brackets in the traditional way, like `foo{}`. If the metric name +// fails the legacy validation check, it will be placed quoted inside the +// brackets: `{"foo"}`. As stated above, the input is assumed to be santized and +// no error will be thrown in this case. +// +// Similar to metric names, if label names conform to the legacy validation +// pattern, they will be unquoted as normal, like `foo{bar="baz"}`. If the label +// name fails the legacy validation check, it will be quoted: +// `foo{"bar"="baz"}`. As stated above, the input is assumed to be santized and +// no error will be thrown in this case. +// // This function fulfills the type 'expfmt.encoder'. // // Note that OpenMetrics requires a final `# EOF` line. Since this function acts @@ -98,7 +110,7 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int if err != nil { return } - n, err = w.WriteString(shortName) + n, err = writeName(w, shortName) written += n if err != nil { return @@ -124,7 +136,7 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int if err != nil { return } - n, err = w.WriteString(shortName) + n, err = writeName(w, shortName) written += n if err != nil { return @@ -303,21 +315,9 @@ func writeOpenMetricsSample( floatValue float64, intValue uint64, useIntValue bool, exemplar *dto.Exemplar, ) (int, error) { - var written int - n, err := w.WriteString(name) - written += n - if err != nil { - return written, err - } - if suffix != "" { - n, err = w.WriteString(suffix) - written += n - if err != nil { - return written, err - } - } - n, err = writeOpenMetricsLabelPairs( - w, metric.Label, additionalLabelName, additionalLabelValue, + written := 0 + n, err := writeOpenMetricsNameAndLabelPairs( + w, name+suffix, metric.Label, additionalLabelName, additionalLabelValue, ) written += n if err != nil { @@ -365,27 +365,58 @@ func writeOpenMetricsSample( return written, nil } -// writeOpenMetricsLabelPairs works like writeOpenMetrics but formats the float -// in OpenMetrics style. -func writeOpenMetricsLabelPairs( +// writeOpenMetricsNameAndLabelPairs works like writeOpenMetricsSample but +// formats the float in OpenMetrics style. +func writeOpenMetricsNameAndLabelPairs( w enhancedWriter, + name string, in []*dto.LabelPair, additionalLabelName string, additionalLabelValue float64, ) (int, error) { - if len(in) == 0 && additionalLabelName == "" { - return 0, nil - } var ( - written int - separator byte = '{' + written int + separator byte = '{' + metricInsideBraces = false ) + + if name != "" { + // If the name does not pass the legacy validity check, we must put the + // metric name inside the braces, quoted. + if !model.IsValidLegacyMetricName(model.LabelValue(name)) { + metricInsideBraces = true + err := w.WriteByte(separator) + written++ + if err != nil { + return written, err + } + separator = ',' + } + + n, err := writeName(w, name) + written += n + if err != nil { + return written, err + } + } + + if len(in) == 0 && additionalLabelName == "" { + if metricInsideBraces { + err := w.WriteByte('}') + written++ + if err != nil { + return written, err + } + } + return written, nil + } + for _, lp := range in { err := w.WriteByte(separator) written++ if err != nil { return written, err } - n, err := w.WriteString(lp.GetName()) + n, err := writeName(w, lp.GetName()) written += n if err != nil { return written, err @@ -451,7 +482,7 @@ func writeExemplar(w enhancedWriter, e *dto.Exemplar) (int, error) { if err != nil { return written, err } - n, err = writeOpenMetricsLabelPairs(w, e.Label, "", 0) + n, err = writeOpenMetricsNameAndLabelPairs(w, "", e.Label, "", 0) written += n if err != nil { return written, err diff --git a/vendor/github.com/prometheus/common/expfmt/text_create.go b/vendor/github.com/prometheus/common/expfmt/text_create.go index 2946b8f..f9b8265 100644 --- a/vendor/github.com/prometheus/common/expfmt/text_create.go +++ b/vendor/github.com/prometheus/common/expfmt/text_create.go @@ -62,6 +62,18 @@ var ( // contains duplicate metrics or invalid metric or label names, the conversion // will result in invalid text format output. // +// If metric names conform to the legacy validation pattern, they will be placed +// outside the brackets in the traditional way, like `foo{}`. If the metric name +// fails the legacy validation check, it will be placed quoted inside the +// brackets: `{"foo"}`. As stated above, the input is assumed to be santized and +// no error will be thrown in this case. +// +// Similar to metric names, if label names conform to the legacy validation +// pattern, they will be unquoted as normal, like `foo{bar="baz"}`. If the label +// name fails the legacy validation check, it will be quoted: +// `foo{"bar"="baz"}`. As stated above, the input is assumed to be santized and +// no error will be thrown in this case. +// // This method fulfills the type 'prometheus.encoder'. func MetricFamilyToText(out io.Writer, in *dto.MetricFamily) (written int, err error) { // Fail-fast checks. @@ -98,7 +110,7 @@ func MetricFamilyToText(out io.Writer, in *dto.MetricFamily) (written int, err e if err != nil { return } - n, err = w.WriteString(name) + n, err = writeName(w, name) written += n if err != nil { return @@ -124,7 +136,7 @@ func MetricFamilyToText(out io.Writer, in *dto.MetricFamily) (written int, err e if err != nil { return } - n, err = w.WriteString(name) + n, err = writeName(w, name) written += n if err != nil { return @@ -280,21 +292,9 @@ func writeSample( additionalLabelName string, additionalLabelValue float64, value float64, ) (int, error) { - var written int - n, err := w.WriteString(name) - written += n - if err != nil { - return written, err - } - if suffix != "" { - n, err = w.WriteString(suffix) - written += n - if err != nil { - return written, err - } - } - n, err = writeLabelPairs( - w, metric.Label, additionalLabelName, additionalLabelValue, + written := 0 + n, err := writeNameAndLabelPairs( + w, name+suffix, metric.Label, additionalLabelName, additionalLabelValue, ) written += n if err != nil { @@ -330,32 +330,64 @@ func writeSample( return written, nil } -// writeLabelPairs converts a slice of LabelPair proto messages plus the -// explicitly given additional label pair into text formatted as required by the -// text format and writes it to 'w'. An empty slice in combination with an empty -// string 'additionalLabelName' results in nothing being written. Otherwise, the -// label pairs are written, escaped as required by the text format, and enclosed -// in '{...}'. The function returns the number of bytes written and any error -// encountered. -func writeLabelPairs( +// writeNameAndLabelPairs converts a slice of LabelPair proto messages plus the +// explicitly given metric name and additional label pair into text formatted as +// required by the text format and writes it to 'w'. An empty slice in +// combination with an empty string 'additionalLabelName' results in nothing +// being written. Otherwise, the label pairs are written, escaped as required by +// the text format, and enclosed in '{...}'. The function returns the number of +// bytes written and any error encountered. If the metric name is not +// legacy-valid, it will be put inside the brackets as well. Legacy-invalid +// label names will also be quoted. +func writeNameAndLabelPairs( w enhancedWriter, + name string, in []*dto.LabelPair, additionalLabelName string, additionalLabelValue float64, ) (int, error) { - if len(in) == 0 && additionalLabelName == "" { - return 0, nil - } var ( - written int - separator byte = '{' + written int + separator byte = '{' + metricInsideBraces = false ) + + if name != "" { + // If the name does not pass the legacy validity check, we must put the + // metric name inside the braces. + if !model.IsValidLegacyMetricName(model.LabelValue(name)) { + metricInsideBraces = true + err := w.WriteByte(separator) + written++ + if err != nil { + return written, err + } + separator = ',' + } + n, err := writeName(w, name) + written += n + if err != nil { + return written, err + } + } + + if len(in) == 0 && additionalLabelName == "" { + if metricInsideBraces { + err := w.WriteByte('}') + written++ + if err != nil { + return written, err + } + } + return written, nil + } + for _, lp := range in { err := w.WriteByte(separator) written++ if err != nil { return written, err } - n, err := w.WriteString(lp.GetName()) + n, err := writeName(w, lp.GetName()) written += n if err != nil { return written, err @@ -462,3 +494,27 @@ func writeInt(w enhancedWriter, i int64) (int, error) { numBufPool.Put(bp) return written, err } + +// writeName writes a string as-is if it complies with the legacy naming +// scheme, or escapes it in double quotes if not. +func writeName(w enhancedWriter, name string) (int, error) { + if model.IsValidLegacyMetricName(model.LabelValue(name)) { + return w.WriteString(name) + } + var written int + var err error + err = w.WriteByte('"') + written++ + if err != nil { + return written, err + } + var n int + n, err = writeEscapedString(w, name, true) + written += n + if err != nil { + return written, err + } + err = w.WriteByte('"') + written++ + return written, err +} diff --git a/vendor/github.com/prometheus/common/model/labels.go b/vendor/github.com/prometheus/common/model/labels.go index ef89563..3317ce2 100644 --- a/vendor/github.com/prometheus/common/model/labels.go +++ b/vendor/github.com/prometheus/common/model/labels.go @@ -97,17 +97,25 @@ var LabelNameRE = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$") // therewith. type LabelName string -// IsValid is true iff the label name matches the pattern of LabelNameRE. This -// method, however, does not use LabelNameRE for the check but a much faster -// hardcoded implementation. +// IsValid returns true iff name matches the pattern of LabelNameRE for legacy +// names, and iff it's valid UTF-8 if NameValidationScheme is set to +// UTF8Validation. For the legacy matching, it does not use LabelNameRE for the +// check but a much faster hardcoded implementation. func (ln LabelName) IsValid() bool { if len(ln) == 0 { return false } - for i, b := range ln { - if !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || (b >= '0' && b <= '9' && i > 0)) { - return false + switch NameValidationScheme { + case LegacyValidation: + for i, b := range ln { + if !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || (b >= '0' && b <= '9' && i > 0)) { + return false + } } + case UTF8Validation: + return utf8.ValidString(string(ln)) + default: + panic(fmt.Sprintf("Invalid name validation scheme requested: %d", NameValidationScheme)) } return true } @@ -164,7 +172,7 @@ func (l LabelNames) String() string { // A LabelValue is an associated value for a LabelName. type LabelValue string -// IsValid returns true iff the string is a valid UTF8. +// IsValid returns true iff the string is a valid UTF-8. func (lv LabelValue) IsValid() bool { return utf8.ValidString(string(lv)) } diff --git a/vendor/github.com/prometheus/common/model/metric.go b/vendor/github.com/prometheus/common/model/metric.go index f8c5eab..0bd29b3 100644 --- a/vendor/github.com/prometheus/common/model/metric.go +++ b/vendor/github.com/prometheus/common/model/metric.go @@ -18,6 +18,77 @@ import ( "regexp" "sort" "strings" + "unicode/utf8" + + dto "github.com/prometheus/client_model/go" + "google.golang.org/protobuf/proto" +) + +var ( + // NameValidationScheme determines the method of name validation to be used by + // all calls to IsValidMetricName() and LabelName IsValid(). Setting UTF-8 mode + // in isolation from other components that don't support UTF-8 may result in + // bugs or other undefined behavior. This value is intended to be set by + // UTF-8-aware binaries as part of their startup. To avoid need for locking, + // this value should be set once, ideally in an init(), before multiple + // goroutines are started. + NameValidationScheme = LegacyValidation + + // NameEscapingScheme defines the default way that names will be + // escaped when presented to systems that do not support UTF-8 names. If the + // Content-Type "escaping" term is specified, that will override this value. + NameEscapingScheme = ValueEncodingEscaping +) + +// ValidationScheme is a Go enum for determining how metric and label names will +// be validated by this library. +type ValidationScheme int + +const ( + // LegacyValidation is a setting that requirets that metric and label names + // conform to the original Prometheus character requirements described by + // MetricNameRE and LabelNameRE. + LegacyValidation ValidationScheme = iota + + // UTF8Validation only requires that metric and label names be valid UTF-8 + // strings. + UTF8Validation +) + +type EscapingScheme int + +const ( + // NoEscaping indicates that a name will not be escaped. Unescaped names that + // do not conform to the legacy validity check will use a new exposition + // format syntax that will be officially standardized in future versions. + NoEscaping EscapingScheme = iota + + // UnderscoreEscaping replaces all legacy-invalid characters with underscores. + UnderscoreEscaping + + // DotsEscaping is similar to UnderscoreEscaping, except that dots are + // converted to `_dot_` and pre-existing underscores are converted to `__`. + DotsEscaping + + // ValueEncodingEscaping prepends the name with `U__` and replaces all invalid + // characters with the unicode value, surrounded by underscores. Single + // underscores are replaced with double underscores. + ValueEncodingEscaping +) + +const ( + // EscapingKey is the key in an Accept or Content-Type header that defines how + // metric and label names that do not conform to the legacy character + // requirements should be escaped when being scraped by a legacy prometheus + // system. If a system does not explicitly pass an escaping parameter in the + // Accept header, the default NameEscapingScheme will be used. + EscapingKey = "escaping" + + // Possible values for Escaping Key: + AllowUTF8 = "allow-utf-8" // No escaping required. + EscapeUnderscores = "underscores" + EscapeDots = "dots" + EscapeValues = "values" ) // MetricNameRE is a regular expression matching valid metric @@ -84,17 +155,302 @@ func (m Metric) FastFingerprint() Fingerprint { return LabelSet(m).FastFingerprint() } -// IsValidMetricName returns true iff name matches the pattern of MetricNameRE. +// IsValidMetricName returns true iff name matches the pattern of MetricNameRE +// for legacy names, and iff it's valid UTF-8 if the UTF8Validation scheme is +// selected. +func IsValidMetricName(n LabelValue) bool { + switch NameValidationScheme { + case LegacyValidation: + return IsValidLegacyMetricName(n) + case UTF8Validation: + if len(n) == 0 { + return false + } + return utf8.ValidString(string(n)) + default: + panic(fmt.Sprintf("Invalid name validation scheme requested: %d", NameValidationScheme)) + } +} + +// IsValidLegacyMetricName is similar to IsValidMetricName but always uses the +// legacy validation scheme regardless of the value of NameValidationScheme. // This function, however, does not use MetricNameRE for the check but a much // faster hardcoded implementation. -func IsValidMetricName(n LabelValue) bool { +func IsValidLegacyMetricName(n LabelValue) bool { if len(n) == 0 { return false } for i, b := range n { - if !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || b == ':' || (b >= '0' && b <= '9' && i > 0)) { + if !isValidLegacyRune(b, i) { return false } } return true } + +// EscapeMetricFamily escapes the given metric names and labels with the given +// escaping scheme. Returns a new object that uses the same pointers to fields +// when possible and creates new escaped versions so as not to mutate the +// input. +func EscapeMetricFamily(v *dto.MetricFamily, scheme EscapingScheme) *dto.MetricFamily { + if v == nil { + return nil + } + + if scheme == NoEscaping { + return v + } + + out := &dto.MetricFamily{ + Help: v.Help, + Type: v.Type, + } + + // If the name is nil, copy as-is, don't try to escape. + if v.Name == nil || IsValidLegacyMetricName(LabelValue(v.GetName())) { + out.Name = v.Name + } else { + out.Name = proto.String(EscapeName(v.GetName(), scheme)) + } + for _, m := range v.Metric { + if !metricNeedsEscaping(m) { + out.Metric = append(out.Metric, m) + continue + } + + escaped := &dto.Metric{ + Gauge: m.Gauge, + Counter: m.Counter, + Summary: m.Summary, + Untyped: m.Untyped, + Histogram: m.Histogram, + TimestampMs: m.TimestampMs, + } + + for _, l := range m.Label { + if l.GetName() == MetricNameLabel { + if l.Value == nil || IsValidLegacyMetricName(LabelValue(l.GetValue())) { + escaped.Label = append(escaped.Label, l) + continue + } + escaped.Label = append(escaped.Label, &dto.LabelPair{ + Name: proto.String(MetricNameLabel), + Value: proto.String(EscapeName(l.GetValue(), scheme)), + }) + continue + } + if l.Name == nil || IsValidLegacyMetricName(LabelValue(l.GetName())) { + escaped.Label = append(escaped.Label, l) + continue + } + escaped.Label = append(escaped.Label, &dto.LabelPair{ + Name: proto.String(EscapeName(l.GetName(), scheme)), + Value: l.Value, + }) + } + out.Metric = append(out.Metric, escaped) + } + return out +} + +func metricNeedsEscaping(m *dto.Metric) bool { + for _, l := range m.Label { + if l.GetName() == MetricNameLabel && !IsValidLegacyMetricName(LabelValue(l.GetValue())) { + return true + } + if !IsValidLegacyMetricName(LabelValue(l.GetName())) { + return true + } + } + return false +} + +const ( + lowerhex = "0123456789abcdef" +) + +// EscapeName escapes the incoming name according to the provided escaping +// scheme. Depending on the rules of escaping, this may cause no change in the +// string that is returned. (Especially NoEscaping, which by definition is a +// noop). This function does not do any validation of the name. +func EscapeName(name string, scheme EscapingScheme) string { + if len(name) == 0 { + return name + } + var escaped strings.Builder + switch scheme { + case NoEscaping: + return name + case UnderscoreEscaping: + if IsValidLegacyMetricName(LabelValue(name)) { + return name + } + for i, b := range name { + if isValidLegacyRune(b, i) { + escaped.WriteRune(b) + } else { + escaped.WriteRune('_') + } + } + return escaped.String() + case DotsEscaping: + // Do not early return for legacy valid names, we still escape underscores. + for i, b := range name { + if b == '_' { + escaped.WriteString("__") + } else if b == '.' { + escaped.WriteString("_dot_") + } else if isValidLegacyRune(b, i) { + escaped.WriteRune(b) + } else { + escaped.WriteRune('_') + } + } + return escaped.String() + case ValueEncodingEscaping: + if IsValidLegacyMetricName(LabelValue(name)) { + return name + } + escaped.WriteString("U__") + for i, b := range name { + if isValidLegacyRune(b, i) { + escaped.WriteRune(b) + } else if !utf8.ValidRune(b) { + escaped.WriteString("_FFFD_") + } else if b < 0x100 { + escaped.WriteRune('_') + for s := 4; s >= 0; s -= 4 { + escaped.WriteByte(lowerhex[b>>uint(s)&0xF]) + } + escaped.WriteRune('_') + } else if b < 0x10000 { + escaped.WriteRune('_') + for s := 12; s >= 0; s -= 4 { + escaped.WriteByte(lowerhex[b>>uint(s)&0xF]) + } + escaped.WriteRune('_') + } + } + return escaped.String() + default: + panic(fmt.Sprintf("invalid escaping scheme %d", scheme)) + } +} + +// lower function taken from strconv.atoi +func lower(c byte) byte { + return c | ('x' - 'X') +} + +// UnescapeName unescapes the incoming name according to the provided escaping +// scheme if possible. Some schemes are partially or totally non-roundtripable. +// If any error is enountered, returns the original input. +func UnescapeName(name string, scheme EscapingScheme) string { + if len(name) == 0 { + return name + } + switch scheme { + case NoEscaping: + return name + case UnderscoreEscaping: + // It is not possible to unescape from underscore replacement. + return name + case DotsEscaping: + name = strings.ReplaceAll(name, "_dot_", ".") + name = strings.ReplaceAll(name, "__", "_") + return name + case ValueEncodingEscaping: + escapedName, found := strings.CutPrefix(name, "U__") + if !found { + return name + } + + var unescaped strings.Builder + TOP: + for i := 0; i < len(escapedName); i++ { + // All non-underscores are treated normally. + if escapedName[i] != '_' { + unescaped.WriteByte(escapedName[i]) + continue + } + i++ + if i >= len(escapedName) { + return name + } + // A double underscore is a single underscore. + if escapedName[i] == '_' { + unescaped.WriteByte('_') + continue + } + // We think we are in a UTF-8 code, process it. + var utf8Val uint + for j := 0; i < len(escapedName); j++ { + // This is too many characters for a utf8 value. + if j > 4 { + return name + } + // Found a closing underscore, convert to a rune, check validity, and append. + if escapedName[i] == '_' { + utf8Rune := rune(utf8Val) + if !utf8.ValidRune(utf8Rune) { + return name + } + unescaped.WriteRune(utf8Rune) + continue TOP + } + r := lower(escapedName[i]) + utf8Val *= 16 + if r >= '0' && r <= '9' { + utf8Val += uint(r) - '0' + } else if r >= 'a' && r <= 'f' { + utf8Val += uint(r) - 'a' + 10 + } else { + return name + } + i++ + } + // Didn't find closing underscore, invalid. + return name + } + return unescaped.String() + default: + panic(fmt.Sprintf("invalid escaping scheme %d", scheme)) + } +} + +func isValidLegacyRune(b rune, i int) bool { + return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || b == ':' || (b >= '0' && b <= '9' && i > 0) +} + +func (e EscapingScheme) String() string { + switch e { + case NoEscaping: + return AllowUTF8 + case UnderscoreEscaping: + return EscapeUnderscores + case DotsEscaping: + return EscapeDots + case ValueEncodingEscaping: + return EscapeValues + default: + panic(fmt.Sprintf("unknown format scheme %d", e)) + } +} + +func ToEscapingScheme(s string) (EscapingScheme, error) { + if s == "" { + return NoEscaping, fmt.Errorf("got empty string instead of escaping scheme") + } + switch s { + case AllowUTF8: + return NoEscaping, nil + case EscapeUnderscores: + return UnderscoreEscaping, nil + case EscapeDots: + return DotsEscaping, nil + case EscapeValues: + return ValueEncodingEscaping, nil + default: + return NoEscaping, fmt.Errorf("unknown format scheme " + s) + } +} diff --git a/vendor/golang.org/x/exp/constraints/constraints.go b/vendor/golang.org/x/exp/constraints/constraints.go new file mode 100644 index 0000000..2c033df --- /dev/null +++ b/vendor/golang.org/x/exp/constraints/constraints.go @@ -0,0 +1,50 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package constraints defines a set of useful constraints to be used +// with type parameters. +package constraints + +// Signed is a constraint that permits any signed integer type. +// If future releases of Go add new predeclared signed integer types, +// this constraint will be modified to include them. +type Signed interface { + ~int | ~int8 | ~int16 | ~int32 | ~int64 +} + +// Unsigned is a constraint that permits any unsigned integer type. +// If future releases of Go add new predeclared unsigned integer types, +// this constraint will be modified to include them. +type Unsigned interface { + ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr +} + +// Integer is a constraint that permits any integer type. +// If future releases of Go add new predeclared integer types, +// this constraint will be modified to include them. +type Integer interface { + Signed | Unsigned +} + +// Float is a constraint that permits any floating-point type. +// If future releases of Go add new predeclared floating-point types, +// this constraint will be modified to include them. +type Float interface { + ~float32 | ~float64 +} + +// Complex is a constraint that permits any complex numeric type. +// If future releases of Go add new predeclared complex numeric types, +// this constraint will be modified to include them. +type Complex interface { + ~complex64 | ~complex128 +} + +// Ordered is a constraint that permits any ordered type: any type +// that supports the operators < <= >= >. +// If future releases of Go add new ordered types, +// this constraint will be modified to include them. +type Ordered interface { + Integer | Float | ~string +} diff --git a/vendor/golang.org/x/sys/unix/mkerrors.sh b/vendor/golang.org/x/sys/unix/mkerrors.sh index c649202..fdcaa97 100644 --- a/vendor/golang.org/x/sys/unix/mkerrors.sh +++ b/vendor/golang.org/x/sys/unix/mkerrors.sh @@ -584,7 +584,7 @@ ccflags="$@" $2 ~ /^KEY_(SPEC|REQKEY_DEFL)_/ || $2 ~ /^KEYCTL_/ || $2 ~ /^PERF_/ || - $2 ~ /^SECCOMP_MODE_/ || + $2 ~ /^SECCOMP_/ || $2 ~ /^SEEK_/ || $2 ~ /^SCHED_/ || $2 ~ /^SPLICE_/ || diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux.go b/vendor/golang.org/x/sys/unix/zerrors_linux.go index a5d3ff8..36bf839 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux.go @@ -1785,6 +1785,8 @@ const ( LANDLOCK_ACCESS_FS_REMOVE_FILE = 0x20 LANDLOCK_ACCESS_FS_TRUNCATE = 0x4000 LANDLOCK_ACCESS_FS_WRITE_FILE = 0x2 + LANDLOCK_ACCESS_NET_BIND_TCP = 0x1 + LANDLOCK_ACCESS_NET_CONNECT_TCP = 0x2 LANDLOCK_CREATE_RULESET_VERSION = 0x1 LINUX_REBOOT_CMD_CAD_OFF = 0x0 LINUX_REBOOT_CMD_CAD_ON = 0x89abcdef @@ -2465,6 +2467,7 @@ const ( PR_MCE_KILL_GET = 0x22 PR_MCE_KILL_LATE = 0x0 PR_MCE_KILL_SET = 0x1 + PR_MDWE_NO_INHERIT = 0x2 PR_MDWE_REFUSE_EXEC_GAIN = 0x1 PR_MPX_DISABLE_MANAGEMENT = 0x2c PR_MPX_ENABLE_MANAGEMENT = 0x2b @@ -2669,8 +2672,9 @@ const ( RTAX_FEATURES = 0xc RTAX_FEATURE_ALLFRAG = 0x8 RTAX_FEATURE_ECN = 0x1 - RTAX_FEATURE_MASK = 0xf + RTAX_FEATURE_MASK = 0x1f RTAX_FEATURE_SACK = 0x2 + RTAX_FEATURE_TCP_USEC_TS = 0x10 RTAX_FEATURE_TIMESTAMP = 0x4 RTAX_HOPLIMIT = 0xa RTAX_INITCWND = 0xb @@ -2913,9 +2917,38 @@ const ( SCM_RIGHTS = 0x1 SCM_TIMESTAMP = 0x1d SC_LOG_FLUSH = 0x100000 + SECCOMP_ADDFD_FLAG_SEND = 0x2 + SECCOMP_ADDFD_FLAG_SETFD = 0x1 + SECCOMP_FILTER_FLAG_LOG = 0x2 + SECCOMP_FILTER_FLAG_NEW_LISTENER = 0x8 + SECCOMP_FILTER_FLAG_SPEC_ALLOW = 0x4 + SECCOMP_FILTER_FLAG_TSYNC = 0x1 + SECCOMP_FILTER_FLAG_TSYNC_ESRCH = 0x10 + SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV = 0x20 + SECCOMP_GET_ACTION_AVAIL = 0x2 + SECCOMP_GET_NOTIF_SIZES = 0x3 + SECCOMP_IOCTL_NOTIF_RECV = 0xc0502100 + SECCOMP_IOCTL_NOTIF_SEND = 0xc0182101 + SECCOMP_IOC_MAGIC = '!' SECCOMP_MODE_DISABLED = 0x0 SECCOMP_MODE_FILTER = 0x2 SECCOMP_MODE_STRICT = 0x1 + SECCOMP_RET_ACTION = 0x7fff0000 + SECCOMP_RET_ACTION_FULL = 0xffff0000 + SECCOMP_RET_ALLOW = 0x7fff0000 + SECCOMP_RET_DATA = 0xffff + SECCOMP_RET_ERRNO = 0x50000 + SECCOMP_RET_KILL = 0x0 + SECCOMP_RET_KILL_PROCESS = 0x80000000 + SECCOMP_RET_KILL_THREAD = 0x0 + SECCOMP_RET_LOG = 0x7ffc0000 + SECCOMP_RET_TRACE = 0x7ff00000 + SECCOMP_RET_TRAP = 0x30000 + SECCOMP_RET_USER_NOTIF = 0x7fc00000 + SECCOMP_SET_MODE_FILTER = 0x1 + SECCOMP_SET_MODE_STRICT = 0x0 + SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP = 0x1 + SECCOMP_USER_NOTIF_FLAG_CONTINUE = 0x1 SECRETMEM_MAGIC = 0x5345434d SECURITYFS_MAGIC = 0x73636673 SEEK_CUR = 0x1 @@ -3075,6 +3108,7 @@ const ( SOL_TIPC = 0x10f SOL_TLS = 0x11a SOL_UDP = 0x11 + SOL_VSOCK = 0x11f SOL_X25 = 0x106 SOL_XDP = 0x11b SOMAXCONN = 0x1000 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_386.go b/vendor/golang.org/x/sys/unix/zerrors_linux_386.go index 4920821..42ff8c3 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_386.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_386.go @@ -281,6 +281,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x40182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x40082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x40082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x800 SIOCATMARK = 0x8905 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go index a0c1e41..dca4360 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go @@ -282,6 +282,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x40182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x40082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x40082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x800 SIOCATMARK = 0x8905 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go b/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go index c639855..5cca668 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go @@ -288,6 +288,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x40182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x40082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x40082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x800 SIOCATMARK = 0x8905 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go index 47cc62e..d8cae6d 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go @@ -278,6 +278,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x40182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x40082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x40082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x800 SIOCATMARK = 0x8905 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go index 27ac4a0..28e39af 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go @@ -275,6 +275,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x40182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x40082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x40082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x800 SIOCATMARK = 0x8905 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go index 5469464..cd66e92 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go @@ -281,6 +281,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x80182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x80082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x80082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x80 SIOCATMARK = 0x40047307 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go index 3adb81d..c1595eb 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go @@ -281,6 +281,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x80182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x80082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x80082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x80 SIOCATMARK = 0x40047307 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go index 2dfe98f..ee9456b 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go @@ -281,6 +281,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x80182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x80082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x80082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x80 SIOCATMARK = 0x40047307 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go index f5398f8..8cfca81 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go @@ -281,6 +281,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x80182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x80082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x80082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x80 SIOCATMARK = 0x40047307 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go index c54f152..60b0deb 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go @@ -336,6 +336,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x80182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x80082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x80082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x800 SIOCATMARK = 0x8905 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go index 76057dc..f90aa72 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go @@ -340,6 +340,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x80182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x80082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x80082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x800 SIOCATMARK = 0x8905 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go index e0c3725..ba9e015 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go @@ -340,6 +340,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x80182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x80082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x80082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x800 SIOCATMARK = 0x8905 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go index 18f2813..07cdfd6 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go @@ -272,6 +272,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x40182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x40082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x40082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x800 SIOCATMARK = 0x8905 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go b/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go index 11619d4..2f1dd21 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go @@ -344,6 +344,9 @@ const ( SCM_TIMESTAMPNS = 0x23 SCM_TXTIME = 0x3d SCM_WIFI_STATUS = 0x29 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x40182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x40082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x40082104 SFD_CLOEXEC = 0x80000 SFD_NONBLOCK = 0x800 SIOCATMARK = 0x8905 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go index 396d994..f40519d 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go @@ -335,6 +335,9 @@ const ( SCM_TIMESTAMPNS = 0x21 SCM_TXTIME = 0x3f SCM_WIFI_STATUS = 0x25 + SECCOMP_IOCTL_NOTIF_ADDFD = 0x80182103 + SECCOMP_IOCTL_NOTIF_ID_VALID = 0x80082102 + SECCOMP_IOCTL_NOTIF_SET_FLAGS = 0x80082104 SFD_CLOEXEC = 0x400000 SFD_NONBLOCK = 0x4000 SF_FP = 0x38 diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go index fcf3ecb..0cc3ce4 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go @@ -448,4 +448,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 450 SYS_CACHESTAT = 451 SYS_FCHMODAT2 = 452 + SYS_MAP_SHADOW_STACK = 453 + SYS_FUTEX_WAKE = 454 + SYS_FUTEX_WAIT = 455 + SYS_FUTEX_REQUEUE = 456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go index f56dc25..856d92d 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go @@ -371,4 +371,7 @@ const ( SYS_CACHESTAT = 451 SYS_FCHMODAT2 = 452 SYS_MAP_SHADOW_STACK = 453 + SYS_FUTEX_WAKE = 454 + SYS_FUTEX_WAIT = 455 + SYS_FUTEX_REQUEUE = 456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go index 974bf24..8d46709 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go @@ -412,4 +412,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 450 SYS_CACHESTAT = 451 SYS_FCHMODAT2 = 452 + SYS_MAP_SHADOW_STACK = 453 + SYS_FUTEX_WAKE = 454 + SYS_FUTEX_WAIT = 455 + SYS_FUTEX_REQUEUE = 456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go index 39a2739..edc1732 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go @@ -315,4 +315,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 450 SYS_CACHESTAT = 451 SYS_FCHMODAT2 = 452 + SYS_MAP_SHADOW_STACK = 453 + SYS_FUTEX_WAKE = 454 + SYS_FUTEX_WAIT = 455 + SYS_FUTEX_REQUEUE = 456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_loong64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_loong64.go index cf9c9d7..445eba2 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_loong64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_loong64.go @@ -309,4 +309,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 450 SYS_CACHESTAT = 451 SYS_FCHMODAT2 = 452 + SYS_MAP_SHADOW_STACK = 453 + SYS_FUTEX_WAKE = 454 + SYS_FUTEX_WAIT = 455 + SYS_FUTEX_REQUEUE = 456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go index 10b7362..adba01b 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go @@ -432,4 +432,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 4450 SYS_CACHESTAT = 4451 SYS_FCHMODAT2 = 4452 + SYS_MAP_SHADOW_STACK = 4453 + SYS_FUTEX_WAKE = 4454 + SYS_FUTEX_WAIT = 4455 + SYS_FUTEX_REQUEUE = 4456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go index cd4d8b4..014c4e9 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go @@ -362,4 +362,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 5450 SYS_CACHESTAT = 5451 SYS_FCHMODAT2 = 5452 + SYS_MAP_SHADOW_STACK = 5453 + SYS_FUTEX_WAKE = 5454 + SYS_FUTEX_WAIT = 5455 + SYS_FUTEX_REQUEUE = 5456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go index 2c0efca..ccc97d7 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go @@ -362,4 +362,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 5450 SYS_CACHESTAT = 5451 SYS_FCHMODAT2 = 5452 + SYS_MAP_SHADOW_STACK = 5453 + SYS_FUTEX_WAKE = 5454 + SYS_FUTEX_WAIT = 5455 + SYS_FUTEX_REQUEUE = 5456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go index a72e31d..ec2b64a 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go @@ -432,4 +432,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 4450 SYS_CACHESTAT = 4451 SYS_FCHMODAT2 = 4452 + SYS_MAP_SHADOW_STACK = 4453 + SYS_FUTEX_WAKE = 4454 + SYS_FUTEX_WAIT = 4455 + SYS_FUTEX_REQUEUE = 4456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc.go index c7d1e37..21a839e 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc.go @@ -439,4 +439,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 450 SYS_CACHESTAT = 451 SYS_FCHMODAT2 = 452 + SYS_MAP_SHADOW_STACK = 453 + SYS_FUTEX_WAKE = 454 + SYS_FUTEX_WAIT = 455 + SYS_FUTEX_REQUEUE = 456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go index f4d4838..c11121e 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go @@ -411,4 +411,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 450 SYS_CACHESTAT = 451 SYS_FCHMODAT2 = 452 + SYS_MAP_SHADOW_STACK = 453 + SYS_FUTEX_WAKE = 454 + SYS_FUTEX_WAIT = 455 + SYS_FUTEX_REQUEUE = 456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go index b64f0e5..909b631 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go @@ -411,4 +411,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 450 SYS_CACHESTAT = 451 SYS_FCHMODAT2 = 452 + SYS_MAP_SHADOW_STACK = 453 + SYS_FUTEX_WAKE = 454 + SYS_FUTEX_WAIT = 455 + SYS_FUTEX_REQUEUE = 456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go index 9571119..e49bed1 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go @@ -316,4 +316,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 450 SYS_CACHESTAT = 451 SYS_FCHMODAT2 = 452 + SYS_MAP_SHADOW_STACK = 453 + SYS_FUTEX_WAKE = 454 + SYS_FUTEX_WAIT = 455 + SYS_FUTEX_REQUEUE = 456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go index f94e943..66017d2 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go @@ -377,4 +377,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 450 SYS_CACHESTAT = 451 SYS_FCHMODAT2 = 452 + SYS_MAP_SHADOW_STACK = 453 + SYS_FUTEX_WAKE = 454 + SYS_FUTEX_WAIT = 455 + SYS_FUTEX_REQUEUE = 456 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go index ba0c2bc..47bab18 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go @@ -390,4 +390,8 @@ const ( SYS_SET_MEMPOLICY_HOME_NODE = 450 SYS_CACHESTAT = 451 SYS_FCHMODAT2 = 452 + SYS_MAP_SHADOW_STACK = 453 + SYS_FUTEX_WAKE = 454 + SYS_FUTEX_WAIT = 455 + SYS_FUTEX_REQUEUE = 456 ) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux.go b/vendor/golang.org/x/sys/unix/ztypes_linux.go index bbf8399..dc0c955 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux.go @@ -174,7 +174,8 @@ type FscryptPolicyV2 struct { Contents_encryption_mode uint8 Filenames_encryption_mode uint8 Flags uint8 - _ [4]uint8 + Log2_data_unit_size uint8 + _ [3]uint8 Master_key_identifier [16]uint8 } @@ -455,60 +456,63 @@ type Ucred struct { } type TCPInfo struct { - State uint8 - Ca_state uint8 - Retransmits uint8 - Probes uint8 - Backoff uint8 - Options uint8 - Rto uint32 - Ato uint32 - Snd_mss uint32 - Rcv_mss uint32 - Unacked uint32 - Sacked uint32 - Lost uint32 - Retrans uint32 - Fackets uint32 - Last_data_sent uint32 - Last_ack_sent uint32 - Last_data_recv uint32 - Last_ack_recv uint32 - Pmtu uint32 - Rcv_ssthresh uint32 - Rtt uint32 - Rttvar uint32 - Snd_ssthresh uint32 - Snd_cwnd uint32 - Advmss uint32 - Reordering uint32 - Rcv_rtt uint32 - Rcv_space uint32 - Total_retrans uint32 - Pacing_rate uint64 - Max_pacing_rate uint64 - Bytes_acked uint64 - Bytes_received uint64 - Segs_out uint32 - Segs_in uint32 - Notsent_bytes uint32 - Min_rtt uint32 - Data_segs_in uint32 - Data_segs_out uint32 - Delivery_rate uint64 - Busy_time uint64 - Rwnd_limited uint64 - Sndbuf_limited uint64 - Delivered uint32 - Delivered_ce uint32 - Bytes_sent uint64 - Bytes_retrans uint64 - Dsack_dups uint32 - Reord_seen uint32 - Rcv_ooopack uint32 - Snd_wnd uint32 - Rcv_wnd uint32 - Rehash uint32 + State uint8 + Ca_state uint8 + Retransmits uint8 + Probes uint8 + Backoff uint8 + Options uint8 + Rto uint32 + Ato uint32 + Snd_mss uint32 + Rcv_mss uint32 + Unacked uint32 + Sacked uint32 + Lost uint32 + Retrans uint32 + Fackets uint32 + Last_data_sent uint32 + Last_ack_sent uint32 + Last_data_recv uint32 + Last_ack_recv uint32 + Pmtu uint32 + Rcv_ssthresh uint32 + Rtt uint32 + Rttvar uint32 + Snd_ssthresh uint32 + Snd_cwnd uint32 + Advmss uint32 + Reordering uint32 + Rcv_rtt uint32 + Rcv_space uint32 + Total_retrans uint32 + Pacing_rate uint64 + Max_pacing_rate uint64 + Bytes_acked uint64 + Bytes_received uint64 + Segs_out uint32 + Segs_in uint32 + Notsent_bytes uint32 + Min_rtt uint32 + Data_segs_in uint32 + Data_segs_out uint32 + Delivery_rate uint64 + Busy_time uint64 + Rwnd_limited uint64 + Sndbuf_limited uint64 + Delivered uint32 + Delivered_ce uint32 + Bytes_sent uint64 + Bytes_retrans uint64 + Dsack_dups uint32 + Reord_seen uint32 + Rcv_ooopack uint32 + Snd_wnd uint32 + Rcv_wnd uint32 + Rehash uint32 + Total_rto uint16 + Total_rto_recoveries uint16 + Total_rto_time uint32 } type CanFilter struct { @@ -551,7 +555,7 @@ const ( SizeofIPv6MTUInfo = 0x20 SizeofICMPv6Filter = 0x20 SizeofUcred = 0xc - SizeofTCPInfo = 0xf0 + SizeofTCPInfo = 0xf8 SizeofCanFilter = 0x8 SizeofTCPRepairOpt = 0x8 ) @@ -3399,7 +3403,7 @@ const ( DEVLINK_PORT_FN_ATTR_STATE = 0x2 DEVLINK_PORT_FN_ATTR_OPSTATE = 0x3 DEVLINK_PORT_FN_ATTR_CAPS = 0x4 - DEVLINK_PORT_FUNCTION_ATTR_MAX = 0x4 + DEVLINK_PORT_FUNCTION_ATTR_MAX = 0x5 ) type FsverityDigest struct { @@ -4183,7 +4187,8 @@ const ( ) type LandlockRulesetAttr struct { - Access_fs uint64 + Access_fs uint64 + Access_net uint64 } type LandlockPathBeneathAttr struct { @@ -5134,7 +5139,7 @@ const ( NL80211_FREQUENCY_ATTR_GO_CONCURRENT = 0xf NL80211_FREQUENCY_ATTR_INDOOR_ONLY = 0xe NL80211_FREQUENCY_ATTR_IR_CONCURRENT = 0xf - NL80211_FREQUENCY_ATTR_MAX = 0x1b + NL80211_FREQUENCY_ATTR_MAX = 0x1c NL80211_FREQUENCY_ATTR_MAX_TX_POWER = 0x6 NL80211_FREQUENCY_ATTR_NO_10MHZ = 0x11 NL80211_FREQUENCY_ATTR_NO_160MHZ = 0xc @@ -5547,7 +5552,7 @@ const ( NL80211_REGDOM_TYPE_CUSTOM_WORLD = 0x2 NL80211_REGDOM_TYPE_INTERSECTION = 0x3 NL80211_REGDOM_TYPE_WORLD = 0x1 - NL80211_REG_RULE_ATTR_MAX = 0x7 + NL80211_REG_RULE_ATTR_MAX = 0x8 NL80211_REKEY_DATA_AKM = 0x4 NL80211_REKEY_DATA_KCK = 0x2 NL80211_REKEY_DATA_KEK = 0x1 diff --git a/vendor/golang.org/x/sys/windows/env_windows.go b/vendor/golang.org/x/sys/windows/env_windows.go index b8ad192..d4577a4 100644 --- a/vendor/golang.org/x/sys/windows/env_windows.go +++ b/vendor/golang.org/x/sys/windows/env_windows.go @@ -37,14 +37,17 @@ func (token Token) Environ(inheritExisting bool) (env []string, err error) { return nil, err } defer DestroyEnvironmentBlock(block) - blockp := unsafe.Pointer(block) - for { - entry := UTF16PtrToString((*uint16)(blockp)) - if len(entry) == 0 { - break + size := unsafe.Sizeof(*block) + for *block != 0 { + // find NUL terminator + end := unsafe.Pointer(block) + for *(*uint16)(end) != 0 { + end = unsafe.Add(end, size) } - env = append(env, entry) - blockp = unsafe.Add(blockp, 2*(len(entry)+1)) + + entry := unsafe.Slice(block, (uintptr(end)-uintptr(unsafe.Pointer(block)))/size) + env = append(env, UTF16ToString(entry)) + block = (*uint16)(unsafe.Add(end, size)) } return env, nil } diff --git a/vendor/golang.org/x/sys/windows/syscall_windows.go b/vendor/golang.org/x/sys/windows/syscall_windows.go index ffb8708..6395a03 100644 --- a/vendor/golang.org/x/sys/windows/syscall_windows.go +++ b/vendor/golang.org/x/sys/windows/syscall_windows.go @@ -125,8 +125,7 @@ func UTF16PtrToString(p *uint16) string { for ptr := unsafe.Pointer(p); *(*uint16)(ptr) != 0; n++ { ptr = unsafe.Pointer(uintptr(ptr) + unsafe.Sizeof(*p)) } - - return string(utf16.Decode(unsafe.Slice(p, n))) + return UTF16ToString(unsafe.Slice(p, n)) } func Getpagesize() int { return 4096 } diff --git a/vendor/modules.txt b/vendor/modules.txt index aa3663b..0b27915 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,7 +1,7 @@ # github.com/DataDog/zstd v1.5.5 ## explicit; go 1.14 github.com/DataDog/zstd -# github.com/NicoNex/echotron/v3 v3.28.0 +# github.com/NicoNex/echotron/v3 v3.29.0 ## explicit; go 1.19 github.com/NicoNex/echotron/v3 # github.com/beorn7/perks v1.0.1 @@ -34,18 +34,20 @@ github.com/cockroachdb/errors/withstack # github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b ## explicit; go 1.16 github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v1.0.0 -## explicit; go 1.19 +# github.com/cockroachdb/pebble v1.1.0 +## explicit; go 1.20 github.com/cockroachdb/pebble github.com/cockroachdb/pebble/internal/arenaskl github.com/cockroachdb/pebble/internal/base github.com/cockroachdb/pebble/internal/batchskl github.com/cockroachdb/pebble/internal/bytealloc github.com/cockroachdb/pebble/internal/cache +github.com/cockroachdb/pebble/internal/constants github.com/cockroachdb/pebble/internal/crc github.com/cockroachdb/pebble/internal/fastrand github.com/cockroachdb/pebble/internal/humanize github.com/cockroachdb/pebble/internal/intern +github.com/cockroachdb/pebble/internal/invalidating github.com/cockroachdb/pebble/internal/invariants github.com/cockroachdb/pebble/internal/keyspan github.com/cockroachdb/pebble/internal/manifest @@ -53,12 +55,15 @@ github.com/cockroachdb/pebble/internal/manual github.com/cockroachdb/pebble/internal/private github.com/cockroachdb/pebble/internal/rangedel github.com/cockroachdb/pebble/internal/rangekey -github.com/cockroachdb/pebble/internal/rate github.com/cockroachdb/pebble/internal/rawalloc github.com/cockroachdb/pebble/internal/testkeys github.com/cockroachdb/pebble/objstorage -github.com/cockroachdb/pebble/objstorage/shared -github.com/cockroachdb/pebble/objstorage/sharedobjcat +github.com/cockroachdb/pebble/objstorage/objstorageprovider +github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing +github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat +github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache +github.com/cockroachdb/pebble/objstorage/remote +github.com/cockroachdb/pebble/rangekey github.com/cockroachdb/pebble/record github.com/cockroachdb/pebble/sstable github.com/cockroachdb/pebble/vfs @@ -75,13 +80,16 @@ github.com/cockroachdb/redact/internal/markers github.com/cockroachdb/redact/internal/redact github.com/cockroachdb/redact/internal/rfmt github.com/cockroachdb/redact/internal/rfmt/fmtsort +# github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 +## explicit; go 1.19 +github.com/cockroachdb/tokenbucket # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew # github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f ## explicit github.com/dgryski/go-rendezvous -# github.com/getsentry/sentry-go v0.26.0 +# github.com/getsentry/sentry-go v0.27.0 ## explicit; go 1.18 github.com/getsentry/sentry-go github.com/getsentry/sentry-go/internal/debug @@ -112,8 +120,8 @@ github.com/golang/snappy # github.com/hjson/hjson-go v3.3.0+incompatible ## explicit github.com/hjson/hjson-go -# github.com/klauspost/compress v1.17.4 -## explicit; go 1.19 +# github.com/klauspost/compress v1.17.7 +## explicit; go 1.20 github.com/klauspost/compress github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 @@ -134,10 +142,10 @@ github.com/pkg/errors ## explicit; go 1.19 github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/internal -# github.com/prometheus/client_model v0.5.0 +# github.com/prometheus/client_model v0.6.0 ## explicit; go 1.19 github.com/prometheus/client_model/go -# github.com/prometheus/common v0.46.0 +# github.com/prometheus/common v0.48.0 ## explicit; go 1.20 github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg @@ -153,10 +161,11 @@ github.com/rogpeppe/go-internal/fmtsort # github.com/sirupsen/logrus v1.9.3 ## explicit; go 1.13 github.com/sirupsen/logrus -# golang.org/x/exp v0.0.0-20240119083558-1b970713d09a +# golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 ## explicit; go 1.20 +golang.org/x/exp/constraints golang.org/x/exp/rand -# golang.org/x/sys v0.16.0 +# golang.org/x/sys v0.17.0 ## explicit; go 1.18 golang.org/x/sys/execabs golang.org/x/sys/unix